diff options
| author | bjormel <bjormel@yahooinc.com> | 2023-10-01 12:23:12 +0000 |
|---|---|---|
| committer | bjormel <bjormel@yahooinc.com> | 2023-10-01 12:23:12 +0000 |
| commit | e9058b555d4dfea2f6c872d9a677e8678b569569 (patch) | |
| tree | fa1b67c6e39712c1e0d9f308b0dd55573b43f913 | |
| parent | 0ad931fa86658904fe9212b014d810236b0e00e4 (diff) | |
| parent | 16030193ec04ee41e98779a3d7ee6a6c1d0d0d6f (diff) | |
Merge branch 'master' into bjormel/aws-main-controller
424 files changed, 9077 insertions, 3613 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index fc4ba131969..9192f163a9b 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -115,6 +115,10 @@ endif() if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11.0) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines") endif() +if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 17.0) + # Turn off dynamic_cast optimization that came with clang 17.0.1 + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-assume-unique-vtables") +endif() if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGOOGLE_PROTOBUF_NO_RDTSC") if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15.0) diff --git a/client/go/internal/admin/jvm/memory.go b/client/go/internal/admin/jvm/memory.go index 8caa1a3be22..2ace1f4ac88 100644 --- a/client/go/internal/admin/jvm/memory.go +++ b/client/go/internal/admin/jvm/memory.go @@ -75,11 +75,11 @@ func ParseJvmMemorySpec(spec string) (result AmountOfMemory, err error) { n, err = fmt.Sscanf(spec, "%d%c", &val, &suffix) if n == 2 && err == nil { switch suffix { - case 'k': + case 'k', 'K': result = KiloBytesOfMemory(val) - case 'm': + case 'm', 'M': result = MegaBytesOfMemory(int(val)) - case 'g': + case 'g', 'G': result = GigaBytesOfMemory(int(val)) default: err = fmt.Errorf("Unknown suffix in JVM memory spec '%s'", spec) diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 1c8f628fe18..572c3095570 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -24,9 +24,9 @@ chalk "^2.4.2" "@babel/compat-data@^7.22.9": - version "7.22.9" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.22.9.tgz#71cdb00a1ce3a329ce4cbec3a44f9fef35669730" - integrity sha512-5UamI7xkUcJ3i9qVDS+KFDEK8/7oJ55/sJMB1Ge7IEapr7KfdfV/HErR+koZwOfd+SgtFKOKRhRakdg++DcJpQ== + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.22.20.tgz#8df6e96661209623f1975d66c35ffca66f3306d0" + integrity sha512-BQYjKbpXjoXwFW5jGqiizJQQT/aC7pFm9Ok1OWssonuguICi264lbgMzRp2ZMmRSlfkX6DsWDDcsrctK8Rwfiw== "@babel/core@^7.1.0", "@babel/core@^7.12.17": version "7.22.9" @@ -70,21 +70,21 @@ json5 "^2.2.3" semver "^6.3.1" -"@babel/core@^7.22.9": - version "7.22.11" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.22.11.tgz#8033acaa2aa24c3f814edaaa057f3ce0ba559c24" - integrity sha512-lh7RJrtPdhibbxndr6/xx0w8+CVlY5FJZiaSz908Fpy+G0xkBFTvwLcKJFF4PJxVfGhVWNebikpWGnOoC71juQ== +"@babel/core@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.22.20.tgz#e3d0eed84c049e2a2ae0a64d27b6a37edec385b7" + integrity sha512-Y6jd1ahLubuYweD/zJH+vvOY141v4f9igNQAQ+MBgq9JlHS2iTsZKn1aMsb3vGccZsXI16VzTBw52Xx0DWmtnA== dependencies: "@ampproject/remapping" "^2.2.0" - "@babel/code-frame" "^7.22.10" - "@babel/generator" "^7.22.10" - "@babel/helper-compilation-targets" "^7.22.10" - "@babel/helper-module-transforms" "^7.22.9" - "@babel/helpers" "^7.22.11" - "@babel/parser" "^7.22.11" - "@babel/template" "^7.22.5" - "@babel/traverse" "^7.22.11" - "@babel/types" "^7.22.11" + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.22.15" + "@babel/helper-compilation-targets" "^7.22.15" + "@babel/helper-module-transforms" "^7.22.20" + "@babel/helpers" "^7.22.15" + "@babel/parser" "^7.22.16" + "@babel/template" "^7.22.15" + "@babel/traverse" "^7.22.20" + "@babel/types" "^7.22.19" convert-source-map "^1.7.0" debug "^4.1.0" gensync "^1.0.0-beta.2" @@ -111,7 +111,7 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" -"@babel/helper-compilation-targets@^7.22.10", "@babel/helper-compilation-targets@^7.22.15": +"@babel/helper-compilation-targets@^7.22.15": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz#0698fc44551a26cf29f18d4662d5bf545a6cfc52" integrity sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw== @@ -133,10 +133,10 @@ lru-cache "^5.1.1" semver "^6.3.1" -"@babel/helper-environment-visitor@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz#f06dd41b7c1f44e1f8da6c4055b41ab3a09a7e98" - integrity sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q== +"@babel/helper-environment-visitor@^7.22.20", "@babel/helper-environment-visitor@^7.22.5": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== "@babel/helper-function-name@^7.22.5": version "7.22.5" @@ -178,6 +178,17 @@ "@babel/helper-split-export-declaration" "^7.22.6" "@babel/helper-validator-identifier" "^7.22.15" +"@babel/helper-module-transforms@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.22.20.tgz#da9edc14794babbe7386df438f3768067132f59e" + integrity sha512-dLT7JVWIUUxKOs1UnJUBR3S70YK+pKX6AbJgB2vMIvEkZkrfJDbYDJesnPshtKV4LhDOR3Oc5YULeDizRek+5A== + dependencies: + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-module-imports" "^7.22.15" + "@babel/helper-simple-access" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/helper-validator-identifier" "^7.22.20" + "@babel/helper-module-transforms@^7.22.5": version "7.22.9" resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.22.9.tgz#92dfcb1fbbb2bc62529024f72d942a8c97142129" @@ -213,17 +224,17 @@ resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== -"@babel/helper-validator-identifier@^7.22.15", "@babel/helper-validator-identifier@^7.22.5": - version "7.22.15" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.15.tgz#601fa28e4cc06786c18912dca138cec73b882044" - integrity sha512-4E/F9IIEi8WR94324mbDUMo074YTheJmd7eZF5vITTeYchqAi6sYXRLHUVsmkdmY4QjfKTcB2jB7dVP3NaBElQ== +"@babel/helper-validator-identifier@^7.22.15", "@babel/helper-validator-identifier@^7.22.19", "@babel/helper-validator-identifier@^7.22.20", "@babel/helper-validator-identifier@^7.22.5": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== "@babel/helper-validator-option@^7.22.15", "@babel/helper-validator-option@^7.22.5": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz#694c30dfa1d09a6534cdfcafbe56789d36aba040" integrity sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA== -"@babel/helpers@^7.22.11", "@babel/helpers@^7.22.15": +"@babel/helpers@^7.22.15": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.22.15.tgz#f09c3df31e86e3ea0b7ff7556d85cdebd47ea6f1" integrity sha512-7pAjK0aSdxOwR+CcYAqgWOGy5dcfvzsTIfFTb2odQqW47MDfv14UaJDY6eng8ylM2EaeKXdxaSWESbkmaQHTmw== @@ -242,11 +253,11 @@ "@babel/types" "^7.22.11" "@babel/highlight@^7.22.13": - version "7.22.13" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.13.tgz#9cda839e5d3be9ca9e8c26b6dd69e7548f0cbf16" - integrity sha512-C/BaXcnnvBCmHTpz/VGZ8jgtE2aYlW4hxDhseJAWZb7gqGM/qtCK6iZUb0TyKFf7BOUsBH7Q7fkRsDRhg1XklQ== + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.20.tgz#4ca92b71d80554b01427815e06f2df965b9c1f54" + integrity sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg== dependencies: - "@babel/helper-validator-identifier" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" chalk "^2.4.2" js-tokens "^4.0.0" @@ -404,7 +415,7 @@ "@babel/parser" "^7.22.15" "@babel/types" "^7.22.15" -"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.15", "@babel/traverse@^7.22.17": +"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.17": version "7.22.17" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.17.tgz#b23c203ab3707e3be816043081b4a994fcacec44" integrity sha512-xK4Uwm0JnAMvxYZxOVecss85WxTEIbTa7bnGyf/+EgCL5Zt3U7htUpEOWv9detPlamGKuRzCqw74xVglDWpPdg== @@ -420,6 +431,22 @@ debug "^4.1.0" globals "^11.1.0" +"@babel/traverse@^7.22.15", "@babel/traverse@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.20.tgz#db572d9cb5c79e02d83e5618b82f6991c07584c9" + integrity sha512-eU260mPZbU7mZ0N+X10pxXhQFMGTeLb9eFS0mxehS8HZp9o1uSnFeWQuG1UPrlxgA7QoUzFhOnilHDp0AXCyHw== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.22.15" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.22.5" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.22.16" + "@babel/types" "^7.22.19" + debug "^4.1.0" + globals "^11.1.0" + "@babel/traverse@^7.22.8": version "7.22.11" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.11.tgz#71ebb3af7a05ff97280b83f05f8865ac94b2027c" @@ -436,7 +463,16 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.10", "@babel/types@^7.22.11", "@babel/types@^7.22.15", "@babel/types@^7.22.17", "@babel/types@^7.22.5", "@babel/types@^7.3.3": +"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.17", "@babel/types@^7.22.19", "@babel/types@^7.22.5": + version "7.22.19" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.19.tgz#7425343253556916e440e662bb221a93ddb75684" + integrity sha512-P7LAw/LbojPzkgp5oznjE6tQEIWbp4PkkfrZDINTro9zgBRtI324/EYsiSI7lhPbpIQ+DCeR2NNmMWANGGfZsg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.19" + to-fast-properties "^2.0.0" + +"@babel/types@^7.22.10", "@babel/types@^7.22.11", "@babel/types@^7.3.3": version "7.22.17" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.17.tgz#f753352c4610ffddf9c8bc6823f9ff03e2303eee" integrity sha512-YSQPHLFtQNE5xN9tHuZnzu8vPr61wVTBZdfv1meex1NBosa4iT05k/Jw06ddJugi4bk7The/oSwQGFcksmEJQg== @@ -1244,22 +1280,40 @@ "@types/babel__template" "*" "@types/babel__traverse" "*" +"@types/babel__core@^7.20.2": + version "7.20.2" + resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.2.tgz#215db4f4a35d710256579784a548907237728756" + integrity sha512-pNpr1T1xLUc2l3xJKuPtsEky3ybxN3m4fJkknfIpTCTfIZCDW57oAg+EfCgIIp2rvCe0Wn++/FfodDS4YXxBwA== + dependencies: + "@babel/parser" "^7.20.7" + "@babel/types" "^7.20.7" + "@types/babel__generator" "*" + "@types/babel__template" "*" + "@types/babel__traverse" "*" + "@types/babel__generator@*": - version "7.6.4" - resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.4.tgz#1f20ce4c5b1990b37900b63f050182d28c2439b7" - integrity sha512-tFkciB9j2K755yrTALxD44McOrk+gfpIpvC3sxHjRawj6PfnQxrse4Clq5y/Rq+G3mrBurMax/lG8Qn2t9mSsg== + version "7.6.5" + resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.5.tgz#281f4764bcbbbc51fdded0f25aa587b4ce14da95" + integrity sha512-h9yIuWbJKdOPLJTbmSpPzkF67e659PbQDba7ifWm5BJ8xTv+sDmS7rFmywkWOvXedGTivCdeGSIIX8WLcRTz8w== dependencies: "@babel/types" "^7.0.0" "@types/babel__template@*": - version "7.4.1" - resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.1.tgz#3d1a48fd9d6c0edfd56f2ff578daed48f36c8969" - integrity sha512-azBFKemX6kMg5Io+/rdGT0dkGreboUVR0Cdm3fz9QJWpaQGJRQXl7C+6hOTCZcMll7KFyEQpgbYI2lHdsS4U7g== + version "7.4.2" + resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.2.tgz#843e9f1f47c957553b0c374481dc4772921d6a6b" + integrity sha512-/AVzPICMhMOMYoSx9MoKpGDKdBRsIXMNByh1PXSZoa+v6ZoLa8xxtsT/uLQ/NJm0XVAWl/BvId4MlDeXJaeIZQ== dependencies: "@babel/parser" "^7.1.0" "@babel/types" "^7.0.0" -"@types/babel__traverse@*", "@types/babel__traverse@^7.0.6": +"@types/babel__traverse@*": + version "7.20.2" + resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.2.tgz#4ddf99d95cfdd946ff35d2b65c978d9c9bf2645d" + integrity sha512-ojlGK1Hsfce93J0+kn3H5R73elidKUaZonirN33GSmgTUMpzI/MIFfSpF3haANe3G1bEBS9/9/QEqwTzwqFsKw== + dependencies: + "@babel/types" "^7.20.7" + +"@types/babel__traverse@^7.0.6": version "7.20.1" resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.1.tgz#dd6f1d2411ae677dcb2db008c962598be31d6acf" integrity sha512-MitHFXnhtgwsGZWtT68URpOvLN4EREih1u3QtQiN4VdAxWKRVvGCSvw/Qth0M0Qq3pJpnGOu5JaM/ydK7OGbqg== @@ -1337,13 +1391,14 @@ "@types/yargs-parser" "*" "@vitejs/plugin-react@^4": - version "4.0.4" - resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.0.4.tgz#31c3f779dc534e045c4b134e7cf7b150af0a7646" - integrity sha512-7wU921ABnNYkETiMaZy7XqpueMnpu5VxvVps13MjmCo+utBdD79sZzrApHawHtVX66cCJQQTXFcjH0y9dSUK8g== + version "4.1.0" + resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.1.0.tgz#e4f56f46fd737c5d386bb1f1ade86ba275fe09bd" + integrity sha512-rM0SqazU9iqPUraQ2JlIvReeaxOoRj6n+PzB1C0cBzIbd8qP336nC39/R9yPi3wVcah7E7j/kdU1uCUqMEU4OQ== dependencies: - "@babel/core" "^7.22.9" + "@babel/core" "^7.22.20" "@babel/plugin-transform-react-jsx-self" "^7.22.5" "@babel/plugin-transform-react-jsx-source" "^7.22.5" + "@types/babel__core" "^7.20.2" react-refresh "^0.14.0" acorn-jsx@^5.3.2: @@ -1725,14 +1780,14 @@ braces@^3.0.2: fill-range "^7.0.1" browserslist@^4.21.9: - version "4.21.10" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.10.tgz#dbbac576628c13d3b2231332cb2ec5a46e015bb0" - integrity sha512-bipEBdZfVH5/pwrvqc+Ub0kUPVfGUhlKxbvfD+z1BDnPEO/X98ruXGA1WP5ASpAFKan7Qr6j736IacbZQuAlKQ== + version "4.21.11" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.11.tgz#35f74a3e51adc4d193dcd76ea13858de7b8fecb8" + integrity sha512-xn1UXOKUz7DjdGlg9RrUr0GGiWzI97UQJnugHtH0OLDfJB7jMgoIkYvRIEO1l9EeEERVqeqLYOcFBW9ldjypbQ== dependencies: - caniuse-lite "^1.0.30001517" - electron-to-chromium "^1.4.477" + caniuse-lite "^1.0.30001538" + electron-to-chromium "^1.4.526" node-releases "^2.0.13" - update-browserslist-db "^1.0.11" + update-browserslist-db "^1.0.13" bser@2.1.1: version "2.1.1" @@ -1791,10 +1846,10 @@ camelcase@^6.2.0: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -caniuse-lite@^1.0.30001517: - version "1.0.30001533" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001533.tgz#1180daeb2518b93c82f19b904d1fefcf82197707" - integrity sha512-9aY/b05NKU4Yl2sbcJhn4A7MsGwR1EPfW/nrqsnqVA0Oq50wpmPaGI+R1Z0UKlUl96oxUkGEOILWtOHck0eCWw== +caniuse-lite@^1.0.30001538: + version "1.0.30001539" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001539.tgz#325a387ab1ed236df2c12dc6cd43a4fff9903a44" + integrity sha512-hfS5tE8bnNiNvEOEkm8HElUHroYwlqMMENEzELymy77+tJ6m+gA2krtHl5hxJaj71OlpC2cHZbdSMX1/YEqEkA== capture-exit@^2.0.0: version "2.0.0" @@ -2124,10 +2179,10 @@ dom-helpers@^5.0.1: "@babel/runtime" "^7.8.7" csstype "^3.0.2" -electron-to-chromium@^1.4.477: - version "1.4.515" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.515.tgz#f5fec9662106ac5752894af221606cf4db443e70" - integrity sha512-VTq6vjk3kCfG2qdzQRd/i9dIyVVm0dbtZIgFzrLgfB73mXDQT2HPKVRc1EoZcAVUv9XhXAu08DWqJuababdGGg== +electron-to-chromium@^1.4.526: + version "1.4.528" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.528.tgz#7c900fd73d9d2e8bb0dab0e301f25f0f4776ef2c" + integrity sha512-UdREXMXzLkREF4jA8t89FQjA8WHI6ssP38PMY4/4KhXFQbtImnghh4GkCgrtiZwLKUKVD2iTVXvDVQjfomEQuA== emittery@^0.13.1: version "0.13.1" @@ -5366,10 +5421,10 @@ untildify@^4.0.0: resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== -update-browserslist-db@^1.0.11: - version "1.0.11" - resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz#9a2a641ad2907ae7b3616506f4b977851db5b940" - integrity sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA== +update-browserslist-db@^1.0.13: + version "1.0.13" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz#3c5e4f5c083661bd38ef64b6328c26ed6c8248c4" + integrity sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg== dependencies: escalade "^3.1.1" picocolors "^1.0.0" diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java index 6b0adb5d079..2dbbc8a5820 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java @@ -5,13 +5,20 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import java.util.logging.Level; -import com.yahoo.yolean.Exceptions; import com.yahoo.vespa.config.util.ConfigUtils; +import com.yahoo.yolean.Exceptions; -import java.io.*; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -208,6 +215,8 @@ public class FilesApplicationFile extends ApplicationFile { } } + @Override public long getSize() { return file.length(); } + @Override public int compareTo(ApplicationFile other) { if (other == this) return 0; diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json new file mode 100644 index 00000000000..2c5be906633 --- /dev/null +++ b/config-model-api/abi-spec.json @@ -0,0 +1,1713 @@ +{ + "com.yahoo.config.application.api.ApplicationFile$MetaData" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>()", + "public void <init>(java.lang.String, java.lang.String)", + "public java.lang.String getStatus()", + "public java.lang.String getMd5()" + ], + "fields" : [ + "public java.lang.String status", + "public java.lang.String md5" + ] + }, + "com.yahoo.config.application.api.ApplicationFile$PathFilter" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract boolean accept(com.yahoo.path.Path)" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.ApplicationFile" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "java.lang.Comparable" + ], + "attributes" : [ + "public", + "abstract" + ], + "methods" : [ + "protected void <init>(com.yahoo.path.Path)", + "public abstract boolean isDirectory()", + "public abstract boolean exists()", + "public abstract java.io.Reader createReader()", + "public abstract java.io.InputStream createInputStream()", + "public abstract com.yahoo.config.application.api.ApplicationFile createDirectory()", + "public abstract com.yahoo.config.application.api.ApplicationFile writeFile(java.io.Reader)", + "public abstract com.yahoo.config.application.api.ApplicationFile appendFile(java.lang.String)", + "public java.util.List listFiles()", + "public abstract java.util.List listFiles(com.yahoo.config.application.api.ApplicationFile$PathFilter)", + "public java.util.List listFiles(boolean)", + "public abstract com.yahoo.config.application.api.ApplicationFile delete()", + "public com.yahoo.path.Path getPath()", + "public java.lang.String toString()", + "public boolean equals(java.lang.Object)", + "protected com.yahoo.path.Path getMetaPath()", + "public abstract com.yahoo.config.application.api.ApplicationFile$MetaData getMetaData()", + "public abstract long getSize()" + ], + "fields" : [ + "public static final java.lang.String ContentStatusNew", + "public static final java.lang.String ContentStatusChanged", + "public static final java.lang.String ContentStatusDeleted", + "protected final com.yahoo.path.Path path" + ] + }, + "com.yahoo.config.application.api.ApplicationMetaData" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String, java.lang.Long, boolean, com.yahoo.config.provision.ApplicationId, java.lang.String, java.lang.Long, long)", + "public void <init>(java.lang.String, java.lang.Long, boolean, com.yahoo.config.provision.ApplicationId, com.yahoo.config.provision.Tags, java.lang.String, java.lang.Long, long)", + "public void <init>(java.lang.String, java.lang.String, java.lang.Long, boolean, com.yahoo.config.provision.ApplicationId, java.lang.String, java.lang.Long, long)", + "public java.lang.String getDeployedByUser()", + "public com.yahoo.config.provision.Tags getTags()", + "public java.lang.String getDeployPath()", + "public com.yahoo.config.provision.ApplicationId getApplicationId()", + "public java.lang.Long getDeployTimestamp()", + "public java.lang.Long getGeneration()", + "public boolean isInternalRedeploy()", + "public java.lang.String getChecksum()", + "public long getPreviousActiveGeneration()", + "public java.lang.String toString()", + "public static com.yahoo.config.application.api.ApplicationMetaData fromJsonString(java.lang.String)", + "public com.yahoo.slime.Slime getSlime()", + "public java.lang.String asJsonString()", + "public byte[] asJsonBytes()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.ApplicationPackage" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract com.yahoo.config.provision.ApplicationId getApplicationId()", + "public abstract java.io.Reader getServices()", + "public abstract java.io.Reader getHosts()", + "public java.util.List getUserIncludeDirs()", + "public void validateIncludeDir(java.lang.String)", + "public abstract java.util.Map getAllExistingConfigDefs()", + "public abstract java.util.List getFiles(com.yahoo.path.Path, java.lang.String, boolean)", + "public java.util.List getFiles(com.yahoo.path.Path, java.lang.String)", + "public java.util.Optional getMajorVersion()", + "public abstract com.yahoo.config.application.api.ApplicationFile getFile(com.yahoo.path.Path)", + "public java.util.List getQueryProfileFiles()", + "public java.util.List getQueryProfileTypeFiles()", + "public java.util.List getPageTemplateFiles()", + "public com.yahoo.config.application.api.ApplicationFile getClientSecurityFile()", + "public abstract java.lang.String getHostSource()", + "public abstract java.lang.String getServicesSource()", + "public abstract java.util.Optional getDeployment()", + "public abstract com.yahoo.config.application.api.DeploymentSpec getDeploymentSpec()", + "public com.yahoo.config.application.api.DeploymentSpec parseDeploymentSpec(boolean)", + "public abstract java.util.Optional getValidationOverrides()", + "public abstract java.util.List getComponentsInfo(com.yahoo.component.Version)", + "public abstract java.io.Reader getRankingExpression(java.lang.String)", + "public static java.lang.String getFileName(java.util.jar.JarEntry)", + "public abstract com.yahoo.config.application.api.ApplicationMetaData getMetaData()", + "public abstract java.io.File getFileReference(com.yahoo.path.Path)", + "public void validateXML()", + "public void validateXMLFor(java.util.Optional)", + "public void writeMetaData()", + "public java.util.Optional getAllocatedHosts()", + "public java.util.Map getFileRegistries()", + "public java.util.Map legacyOverrides()", + "public abstract java.util.Collection getSchemas()", + "public com.yahoo.config.application.api.ApplicationPackage preprocess(com.yahoo.config.provision.Zone, com.yahoo.config.application.api.DeployLogger)" + ], + "fields" : [ + "public static final java.lang.String HOSTS", + "public static final java.lang.String SERVICES", + "public static final com.yahoo.path.Path SCHEMAS_DIR", + "public static final com.yahoo.path.Path SEARCH_DEFINITIONS_DIR", + "public static final java.lang.String COMPONENT_DIR", + "public static final java.lang.String SEARCHCHAINS_DIR", + "public static final java.lang.String DOCPROCCHAINS_DIR", + "public static final java.lang.String PROCESSORCHAINS_DIR", + "public static final java.lang.String ROUTINGTABLES_DIR", + "public static final com.yahoo.path.Path MODELS_DIR", + "public static final com.yahoo.path.Path MODELS_GENERATED_DIR", + "public static final com.yahoo.path.Path MODELS_GENERATED_REPLICATED_DIR", + "public static final com.yahoo.path.Path CONSTANTS_DIR", + "public static final java.lang.String CONFIG_DEFINITIONS_DIR", + "public static final com.yahoo.path.Path QUERY_PROFILES_DIR", + "public static final com.yahoo.path.Path QUERY_PROFILE_TYPES_DIR", + "public static final com.yahoo.path.Path PAGE_TEMPLATES_DIR", + "public static final com.yahoo.path.Path RULES_DIR", + "public static final com.yahoo.path.Path DEPLOYMENT_FILE", + "public static final com.yahoo.path.Path VALIDATION_OVERRIDES", + "public static final com.yahoo.path.Path SECURITY_DIR", + "public static final java.lang.String SD_NAME_SUFFIX", + "public static final java.lang.String RANKEXPRESSION_NAME_SUFFIX", + "public static final java.lang.String RANKPROFILE_NAME_SUFFIX", + "public static final java.lang.String RULES_NAME_SUFFIX", + "public static final java.lang.String EXT_DIR" + ] + }, + "com.yahoo.config.application.api.Bcp$Group" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.util.List, java.time.Duration)", + "public java.util.List members()", + "public java.util.Set memberRegions()", + "public java.time.Duration deadline()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.Bcp$RegionMember" : { + "superClass" : "java.lang.Record", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "record" + ], + "methods" : [ + "public void <init>(com.yahoo.config.provision.RegionName, double)", + "public final java.lang.String toString()", + "public final int hashCode()", + "public final boolean equals(java.lang.Object)", + "public com.yahoo.config.provision.RegionName region()", + "public double fraction()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.Bcp" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.util.List, java.util.Optional)", + "public java.util.Optional defaultDeadline()", + "public java.util.List groups()", + "public com.yahoo.config.application.api.Bcp withGroups(java.util.List)", + "public java.util.Set regions()", + "public boolean isEmpty()", + "public com.yahoo.config.application.api.Bcp orElse(com.yahoo.config.application.api.Bcp)", + "public static com.yahoo.config.application.api.Bcp empty()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.ComponentInfo" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String)", + "public java.lang.String getPathRelativeToAppDir()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.DeployLogger" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract void log(java.util.logging.Level, java.lang.String)", + "public void log(java.util.logging.Level, java.util.function.Supplier)", + "public void log(java.util.logging.Level, java.util.function.Supplier, java.lang.Throwable)", + "public void logApplicationPackage(java.util.logging.Level, java.lang.String)" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.DeploymentInstanceSpec" : { + "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Steps", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(com.yahoo.config.provision.InstanceName, com.yahoo.config.provision.Tags, java.util.List, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, com.yahoo.config.application.api.DeploymentSpec$RevisionTarget, com.yahoo.config.application.api.DeploymentSpec$RevisionChange, com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout, int, int, int, java.util.List, java.util.Optional, java.util.Map, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List, java.util.Map, com.yahoo.config.application.api.Bcp, java.time.Instant)", + "public com.yahoo.config.provision.InstanceName name()", + "public com.yahoo.config.provision.Tags tags()", + "public com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy upgradePolicy()", + "public com.yahoo.config.application.api.DeploymentSpec$RevisionTarget revisionTarget()", + "public com.yahoo.config.application.api.DeploymentSpec$RevisionChange revisionChange()", + "public com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout upgradeRollout()", + "public int minRisk()", + "public int maxRisk()", + "public int maxIdleHours()", + "public java.util.List changeBlocker()", + "public java.util.Optional globalServiceId()", + "public boolean canUpgradeAt(java.time.Instant)", + "public boolean canChangeRevisionAt(java.time.Instant)", + "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", + "public java.util.Map cloudAccounts(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", + "public java.util.Optional hostTTL(com.yahoo.config.provision.Environment, java.util.Optional)", + "public com.yahoo.config.application.api.Notifications notifications()", + "public java.util.List endpoints()", + "public com.yahoo.config.application.api.Bcp bcp()", + "public boolean deploysTo(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", + "public java.util.Map zoneEndpoints(com.yahoo.config.provision.zone.ZoneId)", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.DeploymentSpec$ChangeBlocker" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(boolean, boolean, com.yahoo.config.application.api.TimeWindow)", + "public boolean blocksRevisions()", + "public boolean blocksVersions()", + "public com.yahoo.config.application.api.TimeWindow window()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.DeploymentSpec$DeclaredTest" : { + "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Step", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(com.yahoo.config.provision.RegionName, java.util.Optional)", + "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", + "public boolean isTest()", + "public com.yahoo.config.provision.RegionName region()", + "public java.util.Optional hostTTL()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.DeploymentSpec$DeclaredZone" : { + "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Step", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(com.yahoo.config.provision.Environment)", + "public void <init>(com.yahoo.config.provision.Environment, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Map, java.util.Optional)", + "public com.yahoo.config.provision.Environment environment()", + "public java.util.Optional region()", + "public boolean active()", + "public java.util.Optional testerFlavor()", + "public java.util.List zones()", + "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", + "public boolean isTest()", + "public int hashCode()", + "public boolean equals(java.lang.Object)", + "public java.lang.String toString()", + "public java.util.Optional hostTTL()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.DeploymentSpec$Delay" : { + "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Step", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.time.Duration)", + "public java.time.Duration delay()", + "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.DeploymentSpec$DeprecatedElement" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(int, java.lang.String, java.util.List, java.lang.String)", + "public int majorVersion()", + "public java.lang.String humanReadableString()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.DeploymentSpec$ParallelSteps" : { + "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Steps", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.util.List)", + "public java.time.Duration delay()", + "public boolean isOrdered()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.DeploymentSpec$RevisionChange" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.application.api.DeploymentSpec$RevisionChange[] values()", + "public static com.yahoo.config.application.api.DeploymentSpec$RevisionChange valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionChange whenClear", + "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionChange whenFailing", + "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionChange always" + ] + }, + "com.yahoo.config.application.api.DeploymentSpec$RevisionTarget" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.application.api.DeploymentSpec$RevisionTarget[] values()", + "public static com.yahoo.config.application.api.DeploymentSpec$RevisionTarget valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionTarget next", + "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionTarget latest" + ] + }, + "com.yahoo.config.application.api.DeploymentSpec$Step" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "abstract" + ], + "methods" : [ + "public void <init>()", + "public final boolean concerns(com.yahoo.config.provision.Environment)", + "public abstract boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", + "public java.util.List zones()", + "public java.time.Duration delay()", + "public java.util.List steps()", + "public boolean isTest()", + "public boolean isOrdered()", + "public java.util.Optional hostTTL()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.DeploymentSpec$Steps" : { + "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Step", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.util.List)", + "public java.util.List zones()", + "public java.util.List steps()", + "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", + "public java.time.Duration delay()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy[] values()", + "public static com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy canary", + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy defaultPolicy", + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy conservative" + ] + }, + "com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout[] values()", + "public static com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout separate", + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout leading", + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout simultaneous" + ] + }, + "com.yahoo.config.application.api.DeploymentSpec" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Map, java.util.Optional, java.util.List, java.lang.String, java.util.List)", + "public boolean isEmpty()", + "public java.util.Optional majorVersion()", + "public java.util.List steps()", + "public java.util.Optional athenzDomain()", + "public java.util.Optional athenzService()", + "public java.util.Optional athenzService(com.yahoo.config.provision.InstanceName, com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", + "public com.yahoo.config.provision.CloudAccount cloudAccount(com.yahoo.config.provision.CloudName, com.yahoo.config.provision.InstanceName, com.yahoo.config.provision.zone.ZoneId)", + "public java.util.Map cloudAccounts()", + "public java.util.Optional hostTTL(com.yahoo.config.provision.InstanceName, com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", + "public com.yahoo.config.provision.ZoneEndpoint zoneEndpoint(com.yahoo.config.provision.InstanceName, com.yahoo.config.provision.zone.ZoneId, com.yahoo.config.provision.ClusterSpec$Id)", + "public com.yahoo.config.application.api.Bcp bcp()", + "public java.lang.String xmlForm()", + "public java.util.Optional instance(com.yahoo.config.provision.InstanceName)", + "public com.yahoo.config.application.api.DeploymentInstanceSpec requireInstance(java.lang.String)", + "public com.yahoo.config.application.api.DeploymentInstanceSpec requireInstance(com.yahoo.config.provision.InstanceName)", + "public java.util.List instanceNames()", + "public java.util.List instances()", + "public java.util.List endpoints()", + "public java.util.List deprecatedElements()", + "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.io.Reader)", + "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.lang.String)", + "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.lang.String, boolean)", + "public static java.lang.String toMessageString(java.lang.Throwable)", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public int deployableHashCode()" + ], + "fields" : [ + "public static final com.yahoo.config.application.api.DeploymentSpec empty" + ] + }, + "com.yahoo.config.application.api.Endpoint$Level" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.application.api.Endpoint$Level[] values()", + "public static com.yahoo.config.application.api.Endpoint$Level valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.application.api.Endpoint$Level application", + "public static final enum com.yahoo.config.application.api.Endpoint$Level instance" + ] + }, + "com.yahoo.config.application.api.Endpoint$Target" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(com.yahoo.config.provision.RegionName, com.yahoo.config.provision.InstanceName, int)", + "public com.yahoo.config.provision.RegionName region()", + "public com.yahoo.config.provision.InstanceName instance()", + "public int weight()", + "public java.lang.String toString()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.Endpoint" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String, java.lang.String, com.yahoo.config.application.api.Endpoint$Level, java.util.List)", + "public java.lang.String endpointId()", + "public java.lang.String containerId()", + "public java.util.List regions()", + "public com.yahoo.config.application.api.Endpoint$Level level()", + "public java.util.List targets()", + "public com.yahoo.config.application.api.Endpoint withTargets(java.util.List)", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ + "public static final java.lang.String DEFAULT_ID" + ] + }, + "com.yahoo.config.application.api.FileRegistry$Entry" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String, com.yahoo.config.FileReference)" + ], + "fields" : [ + "public final java.lang.String relativePath", + "public final com.yahoo.config.FileReference reference" + ] + }, + "com.yahoo.config.application.api.FileRegistry" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract com.yahoo.config.FileReference addFile(java.lang.String)", + "public abstract com.yahoo.config.FileReference addUri(java.lang.String)", + "public abstract com.yahoo.config.FileReference addBlob(java.lang.String, java.nio.ByteBuffer)", + "public com.yahoo.config.FileReference addApplicationPackage()", + "public abstract java.util.List export()", + "public java.util.Set asSet()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.Notifications$Role" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.application.api.Notifications$Role[] values()", + "public static com.yahoo.config.application.api.Notifications$Role valueOf(java.lang.String)", + "public static java.lang.String toValue(com.yahoo.config.application.api.Notifications$Role)", + "public static com.yahoo.config.application.api.Notifications$Role fromValue(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.application.api.Notifications$Role author" + ] + }, + "com.yahoo.config.application.api.Notifications$When" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.application.api.Notifications$When[] values()", + "public static com.yahoo.config.application.api.Notifications$When valueOf(java.lang.String)", + "public static java.lang.String toValue(com.yahoo.config.application.api.Notifications$When)", + "public static com.yahoo.config.application.api.Notifications$When fromValue(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.application.api.Notifications$When failing", + "public static final enum com.yahoo.config.application.api.Notifications$When failingCommit" + ] + }, + "com.yahoo.config.application.api.Notifications" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public static com.yahoo.config.application.api.Notifications none()", + "public static com.yahoo.config.application.api.Notifications of(java.util.Map, java.util.Map)", + "public java.util.Set emailAddressesFor(com.yahoo.config.application.api.Notifications$When)", + "public java.util.Set emailRolesFor(com.yahoo.config.application.api.Notifications$When)" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.TimeWindow$LocalDateRange" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public java.util.Optional start()", + "public java.util.Optional end()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.TimeWindow" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public java.util.List days()", + "public java.util.List hours()", + "public java.time.ZoneId zone()", + "public com.yahoo.config.application.api.TimeWindow$LocalDateRange dateRange()", + "public boolean includes(java.time.Instant)", + "public java.lang.String toString()", + "public static com.yahoo.config.application.api.TimeWindow from(java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String)" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.UnparsedConfigDefinition" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract com.yahoo.vespa.config.ConfigDefinition parse()", + "public abstract java.lang.String getUnparsedContent()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.ValidationId" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.application.api.ValidationId[] values()", + "public static com.yahoo.config.application.api.ValidationId valueOf(java.lang.String)", + "public java.lang.String value()", + "public java.lang.String toString()", + "public static java.util.Optional from(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.application.api.ValidationId indexingChange", + "public static final enum com.yahoo.config.application.api.ValidationId indexModeChange", + "public static final enum com.yahoo.config.application.api.ValidationId fieldTypeChange", + "public static final enum com.yahoo.config.application.api.ValidationId clusterSizeReduction", + "public static final enum com.yahoo.config.application.api.ValidationId tensorTypeChange", + "public static final enum com.yahoo.config.application.api.ValidationId resourcesReduction", + "public static final enum com.yahoo.config.application.api.ValidationId contentTypeRemoval", + "public static final enum com.yahoo.config.application.api.ValidationId contentClusterRemoval", + "public static final enum com.yahoo.config.application.api.ValidationId deploymentRemoval", + "public static final enum com.yahoo.config.application.api.ValidationId globalDocumentChange", + "public static final enum com.yahoo.config.application.api.ValidationId configModelVersionMismatch", + "public static final enum com.yahoo.config.application.api.ValidationId skipOldConfigModels", + "public static final enum com.yahoo.config.application.api.ValidationId accessControl", + "public static final enum com.yahoo.config.application.api.ValidationId globalEndpointChange", + "public static final enum com.yahoo.config.application.api.ValidationId zoneEndpointChange", + "public static final enum com.yahoo.config.application.api.ValidationId redundancyIncrease", + "public static final enum com.yahoo.config.application.api.ValidationId redundancyOne", + "public static final enum com.yahoo.config.application.api.ValidationId pagedSettingRemoval", + "public static final enum com.yahoo.config.application.api.ValidationId certificateRemoval" + ] + }, + "com.yahoo.config.application.api.ValidationOverrides$Allow" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(com.yahoo.config.application.api.ValidationId, java.time.Instant)", + "public boolean allows(com.yahoo.config.application.api.ValidationId, java.time.Instant)", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.ValidationOverrides$ValidationException" : { + "superClass" : "java.lang.IllegalArgumentException", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ ], + "fields" : [ ] + }, + "com.yahoo.config.application.api.ValidationOverrides" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.util.List)", + "public void invalid(java.util.Map, java.time.Instant)", + "public void invalid(com.yahoo.config.application.api.ValidationId, java.lang.String, java.time.Instant)", + "public boolean allows(java.lang.String, java.time.Instant)", + "public boolean allows(com.yahoo.config.application.api.ValidationId, java.time.Instant)", + "public boolean validate(java.time.Instant)", + "public java.lang.String xmlForm()", + "public static java.lang.String toAllowMessage(com.yahoo.config.application.api.ValidationId)", + "public static com.yahoo.config.application.api.ValidationOverrides fromXml(java.io.Reader)", + "public static com.yahoo.config.application.api.ValidationOverrides fromXml(java.lang.String)" + ], + "fields" : [ + "public static final com.yahoo.config.application.api.ValidationOverrides empty" + ] + }, + "com.yahoo.config.application.api.xml.DeploymentSpecXmlReader" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(boolean, java.time.Clock)", + "public void <init>()", + "public void <init>(boolean)", + "public com.yahoo.config.application.api.DeploymentSpec read(java.io.Reader)", + "public com.yahoo.config.application.api.DeploymentSpec read(java.lang.String)" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ApplicationClusterEndpoint$AuthMethod" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.model.api.ApplicationClusterEndpoint$AuthMethod[] values()", + "public static com.yahoo.config.model.api.ApplicationClusterEndpoint$AuthMethod valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.model.api.ApplicationClusterEndpoint$AuthMethod mtls", + "public static final enum com.yahoo.config.model.api.ApplicationClusterEndpoint$AuthMethod token" + ] + }, + "com.yahoo.config.model.api.ApplicationClusterEndpoint$Builder" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>()", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$Builder dnsName(com.yahoo.config.model.api.ApplicationClusterEndpoint$DnsName)", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$Builder zoneScope()", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$Builder scope(com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope)", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$Builder sharedRouting()", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$Builder sharedL4Routing()", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$Builder routingMethod(com.yahoo.config.model.api.ApplicationClusterEndpoint$RoutingMethod)", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$Builder weight(int)", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$Builder hosts(java.util.List)", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$Builder clusterId(java.lang.String)", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$Builder authMethod(com.yahoo.config.model.api.ApplicationClusterEndpoint$AuthMethod)", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint build()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ApplicationClusterEndpoint$DnsName" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "java.lang.Comparable" + ], + "attributes" : [ + "public" + ], + "methods" : [ + "public java.lang.String value()", + "public static com.yahoo.config.model.api.ApplicationClusterEndpoint$DnsName sharedL4NameFrom(com.yahoo.config.provision.SystemName, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.config.provision.ApplicationId, java.lang.String)", + "public static com.yahoo.config.model.api.ApplicationClusterEndpoint$DnsName from(java.lang.String)", + "public java.lang.String toString()", + "public int compareTo(com.yahoo.config.model.api.ApplicationClusterEndpoint$DnsName)", + "public bridge synthetic int compareTo(java.lang.Object)" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ApplicationClusterEndpoint$RoutingMethod" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.model.api.ApplicationClusterEndpoint$RoutingMethod[] values()", + "public static com.yahoo.config.model.api.ApplicationClusterEndpoint$RoutingMethod valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.model.api.ApplicationClusterEndpoint$RoutingMethod shared", + "public static final enum com.yahoo.config.model.api.ApplicationClusterEndpoint$RoutingMethod sharedLayer4", + "public static final enum com.yahoo.config.model.api.ApplicationClusterEndpoint$RoutingMethod exclusive" + ] + }, + "com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope[] values()", + "public static com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope application", + "public static final enum com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope global", + "public static final enum com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope zone" + ] + }, + "com.yahoo.config.model.api.ApplicationClusterEndpoint" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$DnsName dnsName()", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope scope()", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$RoutingMethod routingMethod()", + "public int weight()", + "public java.util.List hostNames()", + "public java.lang.String clusterId()", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$AuthMethod authMethod()", + "public java.lang.String toString()", + "public static com.yahoo.config.model.api.ApplicationClusterEndpoint$Builder builder()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ApplicationClusterInfo" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract java.util.List endpoints()", + "public abstract boolean getDeferChangesUntilRestart()", + "public abstract java.lang.String name()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ApplicationInfo" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(com.yahoo.config.provision.ApplicationId, long, com.yahoo.config.model.api.Model)", + "public com.yahoo.config.provision.ApplicationId getApplicationId()", + "public long getGeneration()", + "public com.yahoo.config.model.api.Model getModel()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ApplicationRoles" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String, java.lang.String)", + "public static com.yahoo.config.model.api.ApplicationRoles fromString(java.lang.String, java.lang.String)", + "public java.lang.String applicationContainerRole()", + "public java.lang.String applicationHostRole()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ConfigChangeAction$Type" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.model.api.ConfigChangeAction$Type[] values()", + "public static com.yahoo.config.model.api.ConfigChangeAction$Type valueOf(java.lang.String)", + "public java.lang.String toString()" + ], + "fields" : [ + "public static final enum com.yahoo.config.model.api.ConfigChangeAction$Type RESTART", + "public static final enum com.yahoo.config.model.api.ConfigChangeAction$Type REFEED", + "public static final enum com.yahoo.config.model.api.ConfigChangeAction$Type REINDEX" + ] + }, + "com.yahoo.config.model.api.ConfigChangeAction" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract com.yahoo.config.model.api.ConfigChangeAction$Type getType()", + "public abstract java.lang.String getMessage()", + "public abstract java.util.List getServices()", + "public java.util.Optional validationId()", + "public abstract com.yahoo.config.provision.ClusterSpec$Id clusterId()", + "public boolean ignoreForInternalRedeploy()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ConfigChangeRefeedAction" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.config.model.api.ConfigChangeAction" + ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public com.yahoo.config.model.api.ConfigChangeAction$Type getType()", + "public java.lang.String name()", + "public abstract java.lang.String getDocumentType()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ConfigChangeReindexAction" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.config.model.api.ConfigChangeAction" + ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public com.yahoo.config.model.api.ConfigChangeAction$Type getType()", + "public java.lang.String name()", + "public abstract java.lang.String getDocumentType()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ConfigChangeRestartAction" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.config.model.api.ConfigChangeAction" + ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public com.yahoo.config.model.api.ConfigChangeAction$Type getType()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ConfigDefinitionRepo" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract java.util.Map getConfigDefinitions()", + "public abstract com.yahoo.vespa.config.buildergen.ConfigDefinition get(com.yahoo.vespa.config.ConfigDefinitionKey)" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ConfigDefinitionStore" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract com.yahoo.vespa.config.ConfigDefinition getConfigDefinition(com.yahoo.vespa.config.ConfigDefinitionKey)" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ConfigModelPlugin" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ConfigServerSpec" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract java.lang.String getHostName()", + "public abstract int getConfigServerPort()", + "public abstract int getZooKeeperPort()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ContainerEndpoint" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String, com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope, java.util.List)", + "public void <init>(java.lang.String, com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope, java.util.List, java.util.OptionalInt)", + "public void <init>(java.lang.String, com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope, java.util.List, java.util.OptionalInt, com.yahoo.config.model.api.ApplicationClusterEndpoint$RoutingMethod)", + "public void <init>(java.lang.String, com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope, java.util.List, java.util.OptionalInt, com.yahoo.config.model.api.ApplicationClusterEndpoint$RoutingMethod, com.yahoo.config.model.api.ApplicationClusterEndpoint$AuthMethod)", + "public java.lang.String clusterId()", + "public java.util.List names()", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$Scope scope()", + "public java.util.OptionalInt weight()", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$RoutingMethod routingMethod()", + "public com.yahoo.config.model.api.ApplicationClusterEndpoint$AuthMethod authMethod()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.EndpointCertificateMetadata" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String, java.lang.String, int)", + "public java.lang.String keyName()", + "public java.lang.String certName()", + "public int version()", + "public java.lang.String toString()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.EndpointCertificateSecrets" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String, java.lang.String)", + "public void <init>(java.lang.String, java.lang.String, int)", + "public java.lang.String certificate()", + "public java.lang.String key()", + "public int version()", + "public static com.yahoo.config.model.api.EndpointCertificateSecrets missing(int)", + "public boolean isMissing()" + ], + "fields" : [ + "public static final com.yahoo.config.model.api.EndpointCertificateSecrets MISSING" + ] + }, + "com.yahoo.config.model.api.FileDistribution" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract void startDownload(java.lang.String, int, java.util.Set)" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.HostInfo" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String, java.util.Collection)", + "public java.lang.String getHostname()", + "public java.util.Collection getServices()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.HostProvisioner" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract com.yahoo.config.provision.HostSpec allocateHost(java.lang.String)", + "public abstract java.util.List prepare(com.yahoo.config.provision.ClusterSpec, com.yahoo.config.provision.Capacity, com.yahoo.config.provision.ProvisionLogger)" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.Model" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract com.yahoo.config.ConfigInstance$Builder getConfigInstance(com.yahoo.vespa.config.ConfigKey, com.yahoo.vespa.config.buildergen.ConfigDefinition)", + "public abstract java.util.Set allConfigsProduced()", + "public abstract java.util.Collection getHosts()", + "public abstract java.util.Set allConfigIds()", + "public abstract java.util.Set fileReferences()", + "public abstract com.yahoo.config.provision.AllocatedHosts allocatedHosts()", + "public boolean allowModelVersionMismatch(java.time.Instant)", + "public boolean skipOldConfigModels(java.time.Instant)", + "public com.yahoo.component.Version version()", + "public com.yahoo.component.Version wantedNodeVersion()", + "public com.yahoo.config.model.api.Provisioned provisioned()", + "public java.util.Map documentTypesByCluster()", + "public java.util.Map indexedDocumentTypesByCluster()", + "public java.util.Set applicationClusterInfo()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ModelContext$FeatureFlags" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public double defaultTermwiseLimit()", + "public java.lang.String feedSequencerType()", + "public java.lang.String responseSequencerType()", + "public java.lang.String queryDispatchPolicy()", + "public double queryDispatchWarmup()", + "public int defaultNumResponseThreads()", + "public int mbusNetworkThreads()", + "public int mbusJavaRpcNumTargets()", + "public int mbusJavaEventsBeforeWakeup()", + "public int mbusCppRpcNumTargets()", + "public int mbusCppEventsBeforeWakeup()", + "public int rpcNumTargets()", + "public int rpcEventsBeforeWakeup()", + "public boolean useAsyncMessageHandlingOnSchedule()", + "public double feedConcurrency()", + "public double feedNiceness()", + "public int maxUnCommittedMemory()", + "public boolean sharedStringRepoNoReclaim()", + "public boolean loadCodeAsHugePages()", + "public boolean containerDumpHeapOnShutdownTimeout()", + "public double containerShutdownTimeout()", + "public int heapSizePercentage()", + "public java.util.List allowedAthenzProxyIdentities()", + "public int maxActivationInhibitedOutOfSyncGroups()", + "public java.lang.String jvmOmitStackTraceInFastThrowOption(com.yahoo.config.provision.ClusterSpec$Type)", + "public double resourceLimitDisk()", + "public double resourceLimitMemory()", + "public double minNodeRatioPerGroup()", + "public boolean forwardIssuesAsErrors()", + "public boolean useV8GeoPositions()", + "public int maxCompactBuffers()", + "public java.util.List ignoredHttpUserAgents()", + "public com.yahoo.config.provision.NodeResources$Architecture adminClusterArchitecture()", + "public boolean enableProxyProtocolMixedMode()", + "public java.lang.String logFileCompressionAlgorithm(java.lang.String)", + "public boolean useRestrictedDataPlaneBindings()", + "public boolean enableGlobalPhase()", + "public java.lang.String summaryDecodePolicy()", + "public boolean allowMoreThanOneContentGroupDown(com.yahoo.config.provision.ClusterSpec$Id)", + "public boolean enableConditionalPutRemoveWriteRepair()", + "public boolean enableDataplaneProxy()", + "public boolean enableNestedMultivalueGrouping()", + "public boolean useReconfigurableDispatcher()", + "public int contentLayerMetadataFeatureLevel()", + "public boolean dynamicHeapSize()", + "public java.lang.String unknownConfigDefinition()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ModelContext$ModelFeatureFlag" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "java.lang.annotation.Annotation" + ], + "attributes" : [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods" : [ + "public abstract java.lang.String[] owners()", + "public abstract java.lang.String removeAfter()", + "public abstract java.lang.String comment()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ModelContext$Properties" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract com.yahoo.config.model.api.ModelContext$FeatureFlags featureFlags()", + "public abstract boolean multitenant()", + "public abstract com.yahoo.config.provision.ApplicationId applicationId()", + "public abstract java.util.List configServerSpecs()", + "public abstract com.yahoo.config.provision.HostName loadBalancerName()", + "public abstract java.net.URI ztsUrl()", + "public abstract java.lang.String athenzDnsSuffix()", + "public abstract boolean hostedVespa()", + "public abstract com.yahoo.config.provision.Zone zone()", + "public abstract java.util.Set endpoints()", + "public abstract boolean isBootstrap()", + "public abstract boolean isFirstTimeDeployment()", + "public java.util.Optional endpointCertificateSecrets()", + "public java.util.Optional athenzDomain()", + "public com.yahoo.config.model.api.Quota quota()", + "public java.util.List tenantSecretStores()", + "public java.lang.String jvmGCOptions()", + "public abstract java.lang.String jvmGCOptions(java.util.Optional)", + "public boolean useDedicatedNodeForLogserver()", + "public boolean allowDisableMtls()", + "public java.util.List operatorCertificates()", + "public java.util.List tlsCiphersOverride()", + "public java.util.List zoneDnsSuffixes()", + "public abstract java.util.List environmentVariables()", + "public java.util.Optional cloudAccount()", + "public boolean allowUserFilters()", + "public java.time.Duration endpointConnectionTtl()", + "public java.util.List dataplaneTokens()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ModelContext" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract com.yahoo.config.application.api.ApplicationPackage applicationPackage()", + "public abstract java.util.Optional previousModel()", + "public abstract com.yahoo.config.model.api.HostProvisioner getHostProvisioner()", + "public abstract com.yahoo.config.model.api.Provisioned provisioned()", + "public abstract com.yahoo.config.application.api.DeployLogger deployLogger()", + "public abstract com.yahoo.config.model.api.ConfigDefinitionRepo configDefinitionRepo()", + "public abstract com.yahoo.config.application.api.FileRegistry getFileRegistry()", + "public abstract java.util.concurrent.ExecutorService getExecutor()", + "public java.util.Optional reindexing()", + "public abstract com.yahoo.config.model.api.ModelContext$Properties properties()", + "public java.util.Optional appDir()", + "public abstract com.yahoo.config.model.api.OnnxModelCost onnxModelCost()", + "public java.util.Optional wantedDockerImageRepo()", + "public abstract com.yahoo.component.Version modelVespaVersion()", + "public abstract com.yahoo.component.Version wantedNodeVespaVersion()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ModelCreateResult" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(com.yahoo.config.model.api.Model, java.util.List)", + "public com.yahoo.config.model.api.Model getModel()", + "public java.util.List getConfigChangeActions()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ModelFactory" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract com.yahoo.component.Version version()", + "public abstract com.yahoo.config.model.api.Model createModel(com.yahoo.config.model.api.ModelContext)", + "public abstract com.yahoo.config.model.api.ModelCreateResult createAndValidateModel(com.yahoo.config.model.api.ModelContext, com.yahoo.config.model.api.ValidationParameters)" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ModelState" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract com.yahoo.config.model.api.Model getModel()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.OnnxModelCost$Calculator" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract long aggregatedModelCostInBytes()", + "public abstract void registerModel(com.yahoo.config.application.api.ApplicationFile)", + "public abstract void registerModel(com.yahoo.config.ModelReference)", + "public abstract void registerModel(java.net.URI)" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.OnnxModelCost" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract com.yahoo.config.model.api.OnnxModelCost$Calculator newCalculator(com.yahoo.config.application.api.ApplicationPackage, com.yahoo.config.application.api.DeployLogger)", + "public static com.yahoo.config.model.api.OnnxModelCost disabled()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.PortInfo" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(int, java.util.Collection)", + "public int getPort()", + "public java.util.Collection getTags()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.Provisioned" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>()", + "public void add(com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.config.provision.Capacity)", + "public java.util.Map all()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.Quota" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.util.Optional, java.util.Optional)", + "public static com.yahoo.config.model.api.Quota fromSlime(com.yahoo.slime.Inspector)", + "public com.yahoo.config.model.api.Quota withBudget(java.math.BigDecimal)", + "public com.yahoo.config.model.api.Quota withClusterSize(int)", + "public com.yahoo.slime.Slime toSlime()", + "public void toSlime(com.yahoo.slime.Cursor)", + "public static com.yahoo.config.model.api.Quota unlimited()", + "public java.util.Optional maxClusterSize()", + "public java.util.Optional budgetAsDecimal()", + "public java.util.Optional budget()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.Reindexing$Status" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract java.time.Instant ready()", + "public abstract double speed()", + "public abstract java.lang.String cause()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.Reindexing" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public java.util.Optional status(java.lang.String, java.lang.String)", + "public boolean enabled()" + ], + "fields" : [ + "public static final com.yahoo.config.model.api.Reindexing DISABLED_INSTANCE" + ] + }, + "com.yahoo.config.model.api.ServiceInfo" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String, java.lang.String, java.util.Collection, java.util.Map, java.lang.String, java.lang.String)", + "public java.lang.String getServiceName()", + "public java.lang.String getConfigId()", + "public java.lang.String getServiceType()", + "public java.util.Optional getProperty(java.lang.String)", + "public java.util.Collection getPorts()", + "public java.lang.String getHostName()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.SuperModel" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>()", + "public void <init>(java.util.Map, boolean)", + "public java.util.Map getModelsPerTenant()", + "public java.util.Map getModels()", + "public boolean isComplete()", + "public java.util.List getAllApplicationInfos()", + "public java.util.Optional getApplicationInfo(com.yahoo.config.provision.ApplicationId)", + "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo)", + "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)", + "public com.yahoo.config.model.api.SuperModel cloneAsComplete()", + "public java.util.Set getApplicationIds()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.SuperModelListener" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract void applicationActivated(com.yahoo.config.model.api.SuperModel, com.yahoo.config.model.api.ApplicationInfo)", + "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)", + "public abstract void notifyOfCompleteness(com.yahoo.config.model.api.SuperModel)" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.SuperModelProvider" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract void registerListener(com.yahoo.config.model.api.SuperModelListener)", + "public abstract com.yahoo.config.model.api.SuperModel getSuperModel()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.TenantSecretStore" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String, java.lang.String, java.lang.String)", + "public void <init>(java.lang.String, java.lang.String, java.lang.String, java.util.Optional)", + "public java.lang.String getName()", + "public java.lang.String getAwsId()", + "public java.lang.String getRole()", + "public java.util.Optional getExternalId()", + "public com.yahoo.config.model.api.TenantSecretStore withExternalId(java.lang.String)", + "public java.lang.String toString()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.ValidationParameters$CheckRouting" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.model.api.ValidationParameters$CheckRouting[] values()", + "public static com.yahoo.config.model.api.ValidationParameters$CheckRouting valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.model.api.ValidationParameters$CheckRouting TRUE", + "public static final enum com.yahoo.config.model.api.ValidationParameters$CheckRouting FALSE" + ] + }, + "com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange[] values()", + "public static com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange TRUE", + "public static final enum com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange FALSE" + ] + }, + "com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors[] values()", + "public static com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors TRUE", + "public static final enum com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors FALSE" + ] + }, + "com.yahoo.config.model.api.ValidationParameters" : { + "superClass" : "java.lang.Object", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>()", + "public void <init>(com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors)", + "public void <init>(com.yahoo.config.model.api.ValidationParameters$CheckRouting)", + "public void <init>(com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors, com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange, com.yahoo.config.model.api.ValidationParameters$CheckRouting)", + "public boolean ignoreValidationErrors()", + "public boolean failOnIncompatibleChanges()", + "public boolean checkRouting()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.container.ContainerServiceType" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "enum" + ], + "methods" : [ + "public static com.yahoo.config.model.api.container.ContainerServiceType[] values()", + "public static com.yahoo.config.model.api.container.ContainerServiceType valueOf(java.lang.String)" + ], + "fields" : [ + "public static final enum com.yahoo.config.model.api.container.ContainerServiceType CONTAINER", + "public static final enum com.yahoo.config.model.api.container.ContainerServiceType QRSERVER", + "public static final enum com.yahoo.config.model.api.container.ContainerServiceType CLUSTERCONTROLLER_CONTAINER", + "public static final enum com.yahoo.config.model.api.container.ContainerServiceType LOGSERVER_CONTAINER", + "public static final enum com.yahoo.config.model.api.container.ContainerServiceType METRICS_PROXY_CONTAINER", + "public final java.lang.String serviceName" + ] + } +}
\ No newline at end of file diff --git a/config-model-api/pom.xml b/config-model-api/pom.xml index aaa26a136b5..9bc406452bb 100644 --- a/config-model-api/pom.xml +++ b/config-model-api/pom.xml @@ -106,6 +106,10 @@ </execution> </executions> </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>abi-check-plugin</artifactId> + </plugin> </plugins> </build> </project> diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java index a55ae795d28..97336b2bca0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java @@ -160,6 +160,8 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract MetaData getMetaData(); + public abstract long getSize(); + public static class MetaData { public String status = "unknown"; diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeployLogger.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeployLogger.java index d9ebd902e3e..65e6bc2803a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeployLogger.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeployLogger.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; +import java.util.function.Supplier; import java.util.logging.Level; /** @@ -13,6 +14,10 @@ public interface DeployLogger { /** Log a message unrelated to the application package, e.g. internal error/status. */ void log(Level level, String message); + default void log(Level level, Supplier<String> message) { log(level, message.get()); } + + default void log(Level level, Supplier<String> message, Throwable throwable) { log(level, message); } + /** * Log a message related to the application package. These messages should be actionable by the user, f.ex. to * signal usage of invalid/deprecated syntax diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/package-info.java b/config-model-api/src/main/java/com/yahoo/config/application/api/package-info.java index c1f7d4bd844..0ab9ba3fd63 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/package-info.java @@ -1,5 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage +@PublicApi // Internal public API, not a real public API package com.yahoo.config.application.api; import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.api.annotations.PublicApi; diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/package-info.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/package-info.java index 5ec2ebfa8f9..7c961746eba 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/package-info.java @@ -1,5 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage +@PublicApi // Internal public API, not a real public API package com.yahoo.config.application.api.xml; import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.api.annotations.PublicApi; diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java index 69749ee6f96..0276985d6a6 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java @@ -168,13 +168,7 @@ public class ApplicationClusterEndpoint { return name; } - public static DnsName sharedNameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { - String name = dnsParts(systemName, cluster, applicationId) - .filter(Objects::nonNull) // remove null values that were "default" - .collect(Collectors.joining("--")); - return new DnsName(sanitize(name) + suffix); // Need to sanitize name since it is considered one label - } - + // TODO(mpolden): Remove when config-models < 8.232 are gone public static DnsName sharedL4NameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { String name = dnsParts(systemName, cluster, applicationId) .filter(Objects::nonNull) // remove null values that were "default" diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 37b24f0ac1d..57d013ebd01 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -47,6 +47,7 @@ public interface ModelContext { default Optional<? extends Reindexing> reindexing() { return Optional.empty(); } Properties properties(); default Optional<File> appDir() { return Optional.empty();} + OnnxModelCost onnxModelCost(); /** The Docker image repo we want to use for images for this deployment (optional, will use default if empty) */ default Optional<DockerImage> wantedDockerImageRepo() { return Optional.empty(); } @@ -117,6 +118,8 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean enableNestedMultivalueGrouping() { return false; } @ModelFeatureFlag(owners = {"jonmv"}) default boolean useReconfigurableDispatcher() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default int contentLayerMetadataFeatureLevel() { return 0; } + @ModelFeatureFlag(owners = {"bjorncs"}) default boolean dynamicHeapSize() { return false; } + @ModelFeatureFlag(owners = {"hmusum"}) default String unknownConfigDefinition() { return "log"; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java new file mode 100644 index 00000000000..33ed55ecaef --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.config.model.api; + +import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; + +import java.net.URI; + +/** + * @author bjorncs + */ +public interface OnnxModelCost { + + Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger); + + interface Calculator { + long aggregatedModelCostInBytes(); + void registerModel(ApplicationFile path); + @Deprecated(forRemoval = true) void registerModel(ModelReference ref); // TODO(bjorncs): remove once no longer in use by old config models + void registerModel(URI uri); + } + + static OnnxModelCost disabled() { + return (__, ___) -> new Calculator() { + @Override public long aggregatedModelCostInBytes() { return 0; } + @Override public void registerModel(ApplicationFile path) {} + @SuppressWarnings("removal") @Override public void registerModel(ModelReference ref) {} + @Override public void registerModel(URI uri) {} + }; + } +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/container/package-info.java b/config-model-api/src/main/java/com/yahoo/config/model/api/container/package-info.java index df269bb42e8..fc525b2b589 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/container/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/container/package-info.java @@ -1,5 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage +@PublicApi // Internal public API, not a real public API package com.yahoo.config.model.api.container; import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.api.annotations.PublicApi; diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java index 9560a9658d5..7f6c084863f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java @@ -1,5 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage +@PublicApi // Internal public API, not a real public API package com.yahoo.config.model.api; import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.api.annotations.PublicApi; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 4df7a76031a..a7e8cd52e01 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -18,6 +18,7 @@ import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.api.Reindexing; import com.yahoo.config.model.api.ValidationParameters; @@ -90,6 +91,7 @@ public class DeployState implements ConfigDefinitionStore { private final Provisioned provisioned; private final Reindexing reindexing; private final ExecutorService executor; + private final OnnxModelCost onnxModelCost; public static DeployState createTestState() { return new Builder().build(); @@ -124,7 +126,8 @@ public class DeployState implements ConfigDefinitionStore { boolean accessLoggingEnabledByDefault, Optional<DockerImage> wantedDockerImageRepo, Reindexing reindexing, - Optional<ValidationOverrides> validationOverrides) { + Optional<ValidationOverrides> validationOverrides, + OnnxModelCost onnxModelCost) { this.logger = deployLogger; this.fileRegistry = fileRegistry; this.executor = executor; @@ -152,6 +155,7 @@ public class DeployState implements ConfigDefinitionStore { this.now = now; this.wantedDockerImageRepo = wantedDockerImageRepo; this.reindexing = reindexing; + this.onnxModelCost = onnxModelCost; } public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { @@ -224,7 +228,7 @@ public class DeployState implements ConfigDefinitionStore { // Mapping from key to something that can create a config definition. private Map<ConfigDefinitionKey, UnparsedConfigDefinition> existingConfigDefs = null; - // Cache of config defs for all [def,version] combinations looked up so far. + // Cache of config definitions looked up so far. private final Map<ConfigDefinitionKey, ConfigDefinition> defArchive = new LinkedHashMap<>(); public ApplicationPackage getApplicationPackage() { @@ -305,6 +309,8 @@ public class DeployState implements ConfigDefinitionStore { public Optional<Reindexing> reindexing() { return Optional.ofNullable(reindexing); } + public OnnxModelCost onnxModelCost() { return onnxModelCost; } + public boolean isHostedTenantApplication(ApplicationType type) { boolean isTesterApplication = getProperties().applicationId().instance().isTester(); return isHosted() && type == ApplicationType.DEFAULT && !isTesterApplication; @@ -333,6 +339,7 @@ public class DeployState implements ConfigDefinitionStore { private QueryProfiles queryProfiles = null; private Reindexing reindexing = null; private Optional<ValidationOverrides> validationOverrides = Optional.empty(); + private OnnxModelCost onnxModelCost = OnnxModelCost.disabled(); public Builder() {} @@ -450,6 +457,8 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder onnxModelCost(OnnxModelCost instance) { this.onnxModelCost = instance; return this; } + public DeployState build() { return build(new ValidationParameters()); } @@ -482,7 +491,8 @@ public class DeployState implements ConfigDefinitionStore { accessLoggingEnabledByDefault, wantedDockerImageRepo, reindexing, - validationOverrides); + validationOverrides, + onnxModelCost); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 815c32e3c8f..77356292f9a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -86,6 +86,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean allowUserFilters = true; private List<DataplaneToken> dataplaneTokens; private int contentLayerMetadataFeatureLevel = 0; + private boolean dynamicHeapSize = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -144,6 +145,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only @Override public List<DataplaneToken> dataplaneTokens() { return dataplaneTokens; } @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } + @Override public boolean dynamicHeapSize() { return dynamicHeapSize; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -379,6 +381,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setDynamicHeapSize(boolean b) { this.dynamicHeapSize = b; return this; } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java b/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java index 3d7eafe658f..b59293fbac1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java @@ -2,7 +2,6 @@ package com.yahoo.config.model.producer; import com.yahoo.vespa.config.ConfigDefinitionKey; -import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; import java.util.LinkedHashMap; @@ -16,31 +15,13 @@ import java.util.Set; * @author Ulf Lilleengen */ public class UserConfigRepo { + private final Map<ConfigDefinitionKey, ConfigPayloadBuilder> userConfigsMap; public UserConfigRepo() { this.userConfigsMap = new LinkedHashMap<>(); } - @Override - public UserConfigRepo clone() { - return new UserConfigRepo(copyBuilders(userConfigsMap)); - } - - /** - * Must copy the builder, because the merge method on {@link TreeConfigProducer} might override the row's builders otherwise - */ - private Map<ConfigDefinitionKey, ConfigPayloadBuilder> copyBuilders(Map<ConfigDefinitionKey, ConfigPayloadBuilder> source) { - Map<ConfigDefinitionKey, ConfigPayloadBuilder> ret = new LinkedHashMap<>(); - for (Map.Entry<ConfigDefinitionKey, ConfigPayloadBuilder> e : source.entrySet()) { - ConfigDefinitionKey key = e.getKey(); - ConfigPayloadBuilder sourceVal = e.getValue(); - ConfigPayloadBuilder destVal = new ConfigPayloadBuilder(ConfigPayload.fromBuilder(sourceVal)); - ret.put(key, destVal); - } - return ret; - } - public UserConfigRepo(Map<ConfigDefinitionKey, ConfigPayloadBuilder> map) { this.userConfigsMap = map; } diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index dbcd1cea2fa..342b5f243e7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -488,6 +488,8 @@ public class MockApplicationPackage implements ApplicationPackage { throw new UnsupportedOperationException(); } + @Override public long getSize() { return file.length(); } + @Override public int compareTo(ApplicationFile other) { return this.getPath().getName().compareTo((other).getPath().getName()); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index 7d88985b2d5..f6a022e9930 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -82,7 +82,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } // Commands for summary fields - // TODO: Move to fieldinfo and implement differently. This is not right + // TODO: Move to schemainfo and implement differently for (SummaryField summaryField : schema.getUniqueNamedSummaryFields().values()) { if (summaryField.getTransform().isTeaser()) { addIndexCommand(summaryField.getName(), CMD_DYNTEASER); @@ -90,6 +90,13 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { if (summaryField.getTransform().isBolded()) { addIndexCommand(summaryField.getName(), CMD_HIGHLIGHT); } + + var sourceField = schema.getField(summaryField.getSourceField()); // Take the first as they should all be consistent + if (sourceField != null && sourceField.getMatching().getType().equals(MatchType.GRAM)) { + addIndexCommand(summaryField.getName(), + "ngram " + (sourceField.getMatching().getGramSize().orElse(NGramMatch.DEFAULT_GRAM_SIZE))); + + } } } @@ -452,7 +459,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { iiB.command( new IndexInfoConfig.Indexinfo.Command.Builder() .indexname(fieldSet.getName()) - .command("ngram "+(fieldSetMatching.getGramSize()>0 ? fieldSetMatching.getGramSize() : NGramMatch.DEFAULT_GRAM_SIZE))); + .command("ngram " + fieldSetMatching.getGramSize().orElse(NGramMatch.DEFAULT_GRAM_SIZE))); } else if (fieldSetMatching.getType().equals(MatchType.TEXT)) { } diff --git a/config-model/src/main/java/com/yahoo/schema/document/Matching.java b/config-model/src/main/java/com/yahoo/schema/document/Matching.java index 1fe947d672b..e3f49cb834d 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Matching.java @@ -1,7 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.document; +import com.yahoo.schema.processing.NGramMatch; + import java.io.Serializable; +import java.util.OptionalInt; /** * Defines how a field should be matched. @@ -23,8 +26,8 @@ public class Matching implements Cloneable, Serializable { private boolean algorithmUserSet = false; - /** The gram size is the n in n-gram, or -1 if not set. Should only be set with gram matching. */ - private int gramSize = -1; + /** The gram size is the n in n-gram, or empty if not set. Should only be set with gram matching. */ + private OptionalInt gramSize = OptionalInt.empty(); /** Maximum number of characters to consider when searching in this field. Used for limiting resources, especially in streaming search. */ private Integer maxLength; @@ -67,10 +70,10 @@ public class Matching implements Cloneable, Serializable { public boolean isSuffix() { return algorithm == MatchAlgorithm.SUFFIX; } - /** Returns the gram size, or -1 if not set. Should only be set with gram matching. */ - public int getGramSize() { return gramSize; } + /** Returns the gram size, or empty if not set. Should only be set with gram matching. */ + public OptionalInt getGramSize() { return gramSize; } - public void setGramSize(int gramSize) { this.gramSize=gramSize; } + public void setGramSize(int gramSize) { this.gramSize = OptionalInt.of(gramSize); } /** * Merge data from another matching object @@ -107,10 +110,11 @@ public class Matching implements Cloneable, Serializable { @Override public String toString() { - return type + " matching [" + (type==MatchType.GRAM ? "gram size " + gramSize : "supports " + algorithm) + - "], [exact-terminator "+exactMatchTerminator+"]"; + return type + " matching [" + (type == MatchType.GRAM ? "gram size " + gramSize.orElse(NGramMatch.DEFAULT_GRAM_SIZE) : "supports " + algorithm) + + "], [exact-terminator " + exactMatchTerminator + "]"; } + @Override public Matching clone() { try { return (Matching)super.clone(); @@ -129,7 +133,7 @@ public class Matching implements Cloneable, Serializable { if ( this.exactMatchTerminator == null && other.exactMatchTerminator != null) return false; if ( this.exactMatchTerminator != null && ( ! this.exactMatchTerminator.equals(other.exactMatchTerminator)) ) return false; - if ( gramSize != other.gramSize) return false; + if ( ! gramSize.equals(other.gramSize)) return false; return true; } diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index 7821c101880..6cbdb38b9bc 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -196,6 +196,8 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { return isExtraField; } + public boolean isDocumentField() { return ! isExtraField; } + @Override public boolean isImportedField() { return false; @@ -613,11 +615,8 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { @Override public RankType getRankType() { return this.rankType; } - /** - * Returns the search-time attribute settings of this field or null if none is set. - * - * <p>TODO: Make unmodifiable.</p> - */ + /** Returns the search-time attribute settings of this field or null if none is set. */ + // TODO: Make unmodifiable @Override public Map<String, Attribute> getAttributes() { return attributes; } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java index 985ec8653c7..0537f1704ab 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java @@ -31,9 +31,8 @@ public class IndexingInputs extends Processor { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; - String fieldName = field.getName(); - script = (ScriptExpression)new DefaultToCurrentField(fieldName).convert(script); - script = (ScriptExpression)new EnsureInputExpression(fieldName).convert(script); + script = (ScriptExpression)new DefaultToCurrentField(field).convert(script); + script = (ScriptExpression)new EnsureInputExpression(field).convert(script); if (validate) new VerifyInputExpression(schema, field).visit(script); @@ -43,10 +42,10 @@ public class IndexingInputs extends Processor { private static class DefaultToCurrentField extends ExpressionConverter { - final String fieldName; + final SDField field; - DefaultToCurrentField(String fieldName) { - this.fieldName = fieldName; + DefaultToCurrentField(SDField field) { + this.field = field; } @Override @@ -56,27 +55,28 @@ public class IndexingInputs extends Processor { @Override protected Expression doConvert(Expression exp) { - return new InputExpression(fieldName); + return new InputExpression(field.getName()); } } private static class EnsureInputExpression extends ExpressionConverter { - final String fieldName; + final SDField field; - EnsureInputExpression(String fieldName) { - this.fieldName = fieldName; + EnsureInputExpression(SDField field) { + this.field = field; } @Override protected boolean shouldConvert(Expression exp) { - return exp instanceof StatementExpression; + return exp instanceof StatementExpression + && ( field.isDocumentField() || ( field.getAttribute() != null && field.getAttribute().isMutable())); } @Override protected Expression doConvert(Expression exp) { if (exp.requiredInputType() != null) { - return new StatementExpression(new InputExpression(fieldName), exp); + return new StatementExpression(new InputExpression(field.getName()), exp); } else { return exp; } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java index 3c7e9b4066f..e17b1e46a6e 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; import com.yahoo.vespa.indexinglanguage.expressions.VerificationContext; import com.yahoo.vespa.indexinglanguage.expressions.VerificationException; import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.yolean.Exceptions; import java.util.HashSet; import java.util.Set; @@ -51,7 +52,7 @@ public class IndexingValidation extends Processor { converter.convert(exp); // TODO: stop doing this explicitly when visiting a script does not branch } } catch (VerificationException e) { - fail(schema, field, "For expression '" + e.getExpression() + "': " + e.getMessage()); + fail(schema, field, "For expression '" + e.getExpression() + "': " + Exceptions.toMessageString(e)); } } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java index f1ff910be43..6ec5428156f 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java @@ -31,7 +31,7 @@ public class NGramMatch extends Processor { for (SDField field : schema.allConcreteFields()) { if (field.getMatching().getType().equals(MatchType.GRAM)) implementGramMatch(schema, field, validate); - else if (validate && field.getMatching().getGramSize() >= 0) + else if (validate && field.getMatching().getGramSize().isPresent()) throw new IllegalArgumentException("gram-size can only be set when the matching mode is 'gram'"); } } @@ -40,9 +40,7 @@ public class NGramMatch extends Processor { if (validate && field.doesAttributing() && ! field.doesIndexing()) throw new IllegalArgumentException("gram matching is not supported with attributes, use 'index' in indexing"); - int n = field.getMatching().getGramSize(); - if (n < 0) - n = DEFAULT_GRAM_SIZE; // not set - use default gram size + int n = field.getMatching().getGramSize().orElse(DEFAULT_GRAM_SIZE); if (validate && n == 0) throw new IllegalArgumentException("Illegal gram size in " + field + ": Must be at least 1"); field.getNormalizing().inferCodepoint(); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java index 0362dc39c4c..1627320dc54 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java @@ -15,11 +15,11 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.expressions.ConstantExpression; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.OptimizePredicateExpression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression; import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -112,14 +112,14 @@ public class PredicateProcessor extends Processor { private Expression makeSetPredicateVariablesScript(BooleanIndexDefinition options) { List<Expression> expressions = new ArrayList<>(); - expressions.add(new SetValueExpression(new IntegerFieldValue(options.getArity()))); + expressions.add(new ConstantExpression(new IntegerFieldValue(options.getArity()))); expressions.add(new SetVarExpression("arity")); if (options.hasLowerBound()) { - expressions.add(new SetValueExpression(new LongFieldValue(options.getLowerBound()))); + expressions.add(new ConstantExpression(new LongFieldValue(options.getLowerBound()))); expressions.add(new SetVarExpression("lower_bound")); } if (options.hasUpperBound()) { - expressions.add(new SetValueExpression(new LongFieldValue(options.getUpperBound()))); + expressions.add(new ConstantExpression(new LongFieldValue(options.getUpperBound()))); expressions.add(new SetVarExpression("upper_bound")); } return new StatementExpression(expressions); diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index 7439e65dee6..49cd36e4bc2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -17,9 +17,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; */ public class SummaryField extends Field implements Cloneable, TypedKey { - /** - * A source (field name). - */ + /** A source (field name). */ public static class Source implements Serializable { private final String name; @@ -38,12 +36,8 @@ public class SummaryField extends Field implements Cloneable, TypedKey { @Override public boolean equals(Object obj) { - if (!(obj instanceof Source)) { - return false; - } - Source other = (Source)obj; - return name.equals(other.name) && - override == other.override; + if (!(obj instanceof Source other)) return false; + return name.equals(other.name) && override == other.override; } @Override @@ -67,14 +61,14 @@ public class SummaryField extends Field implements Cloneable, TypedKey { */ private Set<Source> sources = new java.util.LinkedHashSet<>(); - private Set<String> destinations=new java.util.LinkedHashSet<>(); + private Set<String> destinations =new java.util.LinkedHashSet<>(); /** True if this field was defined implicitly */ - private boolean implicit=false; + private boolean implicit = false; /** Creates a summary field with NONE as transform */ public SummaryField(String name, DataType type) { - this(name,type, SummaryTransform.NONE); + this(name, type, SummaryTransform.NONE); } /** Creates a summary field with NONE as transform */ @@ -97,7 +91,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { public boolean isImplicit() { return implicit; } public void setTransform(SummaryTransform transform) { - this.transform=transform; + this.transform = transform; if (SummaryTransform.DYNAMICTEASER.equals(transform) || SummaryTransform.BOLDED.equals(transform)) { // This is the kind of logic we want to have in processing, // but can't because of deriveDocuments mode, which doesn't run @@ -110,9 +104,9 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** Returns the first source field of this, or null if the source field is not present */ public String getSourceField() { - String sourceName=getName(); - if (sources.size()>0) - sourceName=sources.iterator().next().getName(); + String sourceName = getName(); + if ( ! sources.isEmpty()) + sourceName = sources.iterator().next().getName(); return sourceName; } @@ -137,7 +131,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** Returns the first source name of this, or the field name if no source has been set */ public String getSingleSource() { - if (sources.size()==0) return getName(); + if (sources.isEmpty()) return getName(); return sources.iterator().next().getName(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 28ff8dff620..269cb2dfa08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -7,8 +7,8 @@ import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; -import com.yahoo.component.annotation.Inject; import com.yahoo.component.Version; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ValidationOverrides; @@ -197,7 +197,8 @@ public class VespaModelFactory implements ModelFactory { .zone(zone) .now(clock.instant()) .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()) - .wantedDockerImageRepo(modelContext.wantedDockerImageRepo()); + .wantedDockerImageRepo(modelContext.wantedDockerImageRepo()) + .onnxModelCost(modelContext.onnxModelCost()); modelContext.previousModel().ifPresent(builder::previousModel); modelContext.reindexing().ifPresent(builder::reindexing); return builder.build(validationParameters); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index aa3b8b3b821..d10a631fb90 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -159,6 +159,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC builder.consumer.add(toConsumerBuilder(MetricsConsumer.defaultConsumer)); builder.consumer.add(toConsumerBuilder(newDefaultConsumer())); + if (isHostedVespa()) builder.consumer.add(toConsumerBuilder(MetricsConsumer.vespa9)); getAdmin() .map(Admin::getAmendedMetricsConsumers) .map(consumers -> consumers.stream().map(ConsumersConfigGenerator::toConsumerBuilder).toList()) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index cfe3c01e03a..987812f11ad 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.admin.monitoring; import ai.vespa.metrics.set.Metric; import ai.vespa.metrics.set.MetricSet; +import ai.vespa.metrics.set.Vespa9VespaMetricSet; import ai.vespa.metricsproxy.core.VespaMetrics; import ai.vespa.metricsproxy.http.ValuesFetcher; @@ -41,6 +42,9 @@ public class MetricsConsumer { public static final MetricsConsumer vespaCloud = consumer("vespa-cloud", vespaMetricSet, systemMetricSet, networkMetricSet); + public static final MetricsConsumer vespa9 = + consumer("Vespa9", Vespa9VespaMetricSet.vespa9vespaMetricSet, systemMetricSet, networkMetricSet); + private final String id; private final MetricSet metricSet; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java new file mode 100644 index 00000000000..f87cecb58fe --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; + +import java.util.logging.Level; + +/** + * Validates that the container node flavour has enough resources to run configured ONNX models. + * + * @author bjorncs + */ +public class JvmHeapSizeValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState ds) { + if (!ds.featureFlags().dynamicHeapSize()) return; + if (!ds.isHostedTenantApplication(model.getAdmin().getApplicationType())) return; + + model.getContainerClusters().forEach((clusterId, appCluster) -> { + var mp = appCluster.getMemoryPercentage().orElse(null); + if (mp == null) return; + if (mp.availableMemoryGb().isEmpty()) { + ds.getDeployLogger().log(Level.FINE, "Host resources unknown or percentage overridden with 'allocated-memory'"); + return; + } + long jvmModelCost = appCluster.onnxModelCost().aggregatedModelCostInBytes(); + if (jvmModelCost > 0) { + int percentLimit = 15; + double gbLimit = 0.6; + double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); + double modelCostGb = jvmModelCost / (1024D * 1024 * 1024); + ds.getDeployLogger().log(Level.FINE, () -> "JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB" + .formatted(mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb)); + if (mp.percentage() < percentLimit) { + throw new IllegalArgumentException( + ("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") + .formatted(clusterId, mp.percentage(), percentLimit, modelCostGb)); + } + if (availableMemoryGb < gbLimit) { + throw new IllegalArgumentException( + ("Allocated memory to JVM in cluster '%s' is too low (%.2fGB < %.2fGB). " + + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") + .formatted(clusterId, availableMemoryGb, gbLimit, modelCostGb)); + } + } + }); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java new file mode 100644 index 00000000000..d9dd3729bd3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; + +/** + * Validates that config using s3:// urls is used in public system and with nodes that are exclusive. + * + * @author hmusum + */ +public class UrlConfigValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState state) { + if (! state.isHostedTenantApplication(model.getAdmin().getApplicationType())) return; + + model.getContainerClusters().forEach((__, cluster) -> { + var isExclusive = hasExclusiveNodes(model, cluster); + validateS3UlsInConfig(state, cluster, isExclusive); + }); + } + + private static boolean hasExclusiveNodes(VespaModel model, ApplicationContainerCluster cluster) { + return model.hostSystem().getHosts() + .stream() + .flatMap(hostResource -> hostResource.spec().membership().stream()) + .filter(membership -> membership.cluster().id().equals(cluster.id())) + .anyMatch(membership -> membership.cluster().isExclusive()); + } + + private static void validateS3UlsInConfig(DeployState state, ApplicationContainerCluster cluster, boolean isExclusive) { + if (hasS3UrlInConfig(cluster)) { + // TODO: Would be even better if we could add which config/field the url is set for in the error message + String message = "Found s3:// urls in config for container cluster " + cluster.getName(); + if ( ! state.zone().system().isPublic()) + throw new IllegalArgumentException(message + ". This is only supported in public systems"); + else if ( ! isExclusive) + throw new IllegalArgumentException(message + ". Nodes in the cluster need to be 'exclusive'," + + " see https://cloud.vespa.ai/en/reference/services#nodes"); + } + } + + private static boolean hasS3UrlInConfig(ApplicationContainerCluster cluster) { + return cluster.userConfiguredUrls().all().stream() + .anyMatch(url -> url.startsWith("s3://")); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 53a553ee624..30aafe67be7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -87,6 +87,8 @@ public class Validation { new AccessControlFilterExcludeValidator().validate(model, deployState); new CloudUserFilterValidator().validate(model, deployState); new CloudHttpConnectorValidator().validate(model, deployState); + new UrlConfigValidator().validate(model, deployState); + new JvmHeapSizeValidator().validate(model, deployState); additionalValidators.forEach(v -> v.validate(model, deployState)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java index bbfa939f8a3..f265f2d09a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java @@ -7,6 +7,7 @@ import com.yahoo.schema.document.Matching; import com.yahoo.schema.document.MatchType; import com.yahoo.schema.document.NormalizeLevel; import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.processing.NGramMatch; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -89,7 +90,7 @@ public class IndexingScriptChangeMessageBuilder { MatchType type = matching.getType(); String retval = type.getName(); if (type == MatchType.GRAM) { - retval += " (size " + matching.getGramSize() + ")"; + retval += " (size " + matching.getGramSize().orElse(NGramMatch.DEFAULT_GRAM_SIZE) + ")"; } return retval; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java index 7501f6162c7..9ecd359f90d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java @@ -7,11 +7,12 @@ import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; -import com.yahoo.vespa.model.container.component.HuggingFaceEmbedder; -import com.yahoo.vespa.model.container.component.HuggingFaceTokenizer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.BertEmbedder; import com.yahoo.vespa.model.container.component.ColBertEmbedder; import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.HuggingFaceEmbedder; +import com.yahoo.vespa.model.container.component.HuggingFaceTokenizer; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; @@ -35,19 +36,20 @@ public class DomComponentBuilder extends VespaDomBuilder.DomConfigProducerBuilde @Override protected Component<? super Component<?, ?>, ?> doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, Element spec) { - var component = buildComponent(spec, deployState); + var component = buildComponent(spec, deployState, ancestor); addChildren(deployState, ancestor, spec, component); return component; } - private Component<? super Component<?, ?>, ?> buildComponent(Element spec, DeployState state) { + private Component<? super Component<?, ?>, ?> buildComponent( + Element spec, DeployState state, TreeConfigProducer<AnyConfigProducer> ancestor) { if (spec.hasAttribute("type")) { var type = spec.getAttribute("type"); return switch (type) { - case "hugging-face-embedder" -> new HuggingFaceEmbedder(spec, state); + case "hugging-face-embedder" -> new HuggingFaceEmbedder((ApplicationContainerCluster)ancestor, spec, state); case "hugging-face-tokenizer" -> new HuggingFaceTokenizer(spec, state); - case "bert-embedder" -> new BertEmbedder(spec, state); - case "colbert-embedder" -> new ColBertEmbedder(spec, state); + case "colbert-embedder" -> new ColBertEmbedder((ApplicationContainerCluster)ancestor, spec, state); + case "bert-embedder" -> new BertEmbedder((ApplicationContainerCluster)ancestor, spec, state); default -> throw new IllegalArgumentException("Unknown component type '%s'".formatted(type)); }; } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 0795fdf41d6..762c670039c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.clients; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; @@ -10,6 +11,7 @@ import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.UserBindingPattern; +import org.w3c.dom.Element; import java.nio.file.Path; import java.util.Collection; @@ -29,10 +31,11 @@ public class ContainerDocumentApi { private final boolean ignoreUndefinedFields; - public ContainerDocumentApi(ContainerCluster<?> cluster, HandlerOptions handlerOptions, boolean ignoreUndefinedFields, Set<Integer> portOverride) { + public ContainerDocumentApi(DeployState ds, ContainerCluster<?> cluster, HandlerOptions handlerOptions, + boolean ignoreUndefinedFields, Set<Integer> portOverride) { this.ignoreUndefinedFields = ignoreUndefinedFields; addRestApiHandler(cluster, handlerOptions, portOverride); - addFeedHandler(cluster, handlerOptions, portOverride); + addFeedHandler(ds, cluster, handlerOptions, portOverride); addVespaClientContainerBundle(cluster); } @@ -40,9 +43,9 @@ public class ContainerDocumentApi { c.addPlatformBundle(VESPACLIENT_CONTAINER_BUNDLE); } - private static void addFeedHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions, Set<Integer> portOverride) { + private static void addFeedHandler(DeployState ds, ContainerCluster<?> cluster, HandlerOptions handlerOptions, Set<Integer> portOverride) { String bindingSuffix = ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"; - var executor = new Threadpool("feedapi-handler", handlerOptions.feedApiThreadpoolOptions); + var executor = new Threadpool(ds, "feedapi-handler", handlerOptions.feedApiThreadpoolOptions); var handler = newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", bindingSuffix, handlerOptions, executor, portOverride); cluster.addComponent(handler); @@ -104,9 +107,9 @@ public class ContainerDocumentApi { public static final class HandlerOptions { private final Collection<String> bindings; - private final ContainerThreadpool.UserOptions feedApiThreadpoolOptions; + private final Element feedApiThreadpoolOptions; - public HandlerOptions(Collection<String> bindings, ContainerThreadpool.UserOptions feedApiThreadpoolOptions) { + public HandlerOptions(Collection<String> bindings, Element feedApiThreadpoolOptions) { this.bindings = Collections.unmodifiableCollection(bindings); this.feedApiThreadpoolOptions = feedApiThreadpoolOptions; } @@ -114,9 +117,7 @@ public class ContainerDocumentApi { private static class Threadpool extends ContainerThreadpool { - Threadpool(String name, ContainerThreadpool.UserOptions threadpoolOptions) { - super(name, threadpoolOptions); - } + Threadpool(DeployState ds, String name, Element xml) { super(ds, name, xml); } @Override protected void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index b9021912244..ac679cc406c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -8,14 +8,14 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ApplicationClusterInfo; -import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.di.config.ApplicationBundlesConfig; @@ -43,10 +43,12 @@ import com.yahoo.vespa.model.filedistribution.UserConfiguredFiles; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.stream.Collectors; import static com.yahoo.vespa.model.container.docproc.DocprocChains.DOCUMENT_TYPE_MANAGER_CLASS; @@ -82,6 +84,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); private final Set<String> previousHosts; + private final OnnxModelCost.Calculator onnxModelCost; + private final DeployLogger logger; private ContainerModelEvaluation modelEvaluation; @@ -92,6 +96,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private int zookeeperSessionTimeoutSeconds = 30; private final int transport_events_before_wakeup; private final int transport_connections_per_target; + private final boolean dynamicHeapSize; /** The heap size % of total memory available to the JVM process. */ private final int heapSizePercentageOfAvailableMemory; @@ -100,9 +105,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private List<ApplicationClusterEndpoint> endpoints = List.of(); + private final UserConfiguredUrls userConfiguredUrls = new UserConfiguredUrls(); + public ApplicationContainerCluster(TreeConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) { super(parent, configSubId, clusterId, deployState, true, 10); this.tlsClientAuthority = deployState.tlsClientAuthority(); + dynamicHeapSize = deployState.featureFlags().dynamicHeapSize(); previousHosts = Collections.unmodifiableSet(deployState.getPreviousModel().stream() .map(Model::allocatedHosts) .map(AllocatedHosts::getHosts) @@ -125,8 +133,13 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat heapSizePercentageOfAvailableMemory = deployState.featureFlags().heapSizePercentage() > 0 ? Math.min(99, deployState.featureFlags().heapSizePercentage()) : defaultHeapSizePercentageOfAvailableMemory; + onnxModelCost = deployState.onnxModelCost().newCalculator( + deployState.getApplicationPackage(), deployState.getDeployLogger()); + logger = deployState.getDeployLogger(); } + public UserConfiguredUrls userConfiguredUrls() { return userConfiguredUrls; } + @Override protected void doPrepare(DeployState deployState) { super.doPrepare(deployState); @@ -147,7 +160,10 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat if (containers.isEmpty()) return; // Files referenced from user configs to all components. - UserConfiguredFiles files = new UserConfiguredFiles(deployState.getFileRegistry(), deployState.getDeployLogger()); + UserConfiguredFiles files = new UserConfiguredFiles(deployState.getFileRegistry(), + deployState.getDeployLogger(), + deployState.featureFlags(), + userConfiguredUrls); for (Component<?, ?> component : getAllComponents()) { files.register(component); } @@ -182,19 +198,25 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public void setMemoryPercentage(Integer memoryPercentage) { this.memoryPercentage = memoryPercentage; } @Override - public Optional<Integer> getMemoryPercentage() { - if (memoryPercentage != null) return Optional.of(memoryPercentage); + public Optional<JvmMemoryPercentage> getMemoryPercentage() { + if (memoryPercentage != null) return Optional.of(JvmMemoryPercentage.of(memoryPercentage)); if (isHostedVespa()) { int availableMemoryPercentage = getHostClusterId().isPresent() ? heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster : heapSizePercentageOfAvailableMemory; - if (getContainers().isEmpty()) return Optional.of(availableMemoryPercentage); // Node memory is not known + if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(availableMemoryPercentage)); // Node memory is not known // Node memory is known so convert available memory percentage to node memory percentage - double totalMemory = getContainers().get(0).getHostResource().realResources().memoryGb(); - double availableMemory = totalMemory - Host.memoryOverheadGb; - return Optional.of((int) (availableMemory / totalMemory * availableMemoryPercentage)); + double totalMemory = dynamicHeapSize + ? getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow() + : getContainers().get(0).getHostResource().realResources().memoryGb(); + double jvmHeapDeductionGb = dynamicHeapSize ? onnxModelCost.aggregatedModelCostInBytes() / (1024D * 1024 * 1024) : 0; + double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - jvmHeapDeductionGb); + int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); + logger.log(Level.FINE, () -> "memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" + .formatted(memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); + return Optional.of(JvmMemoryPercentage.of(memoryPercentage, availableMemory)); } return Optional.empty(); } @@ -203,49 +225,23 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private void createEndpoints(DeployState deployState) { if (!deployState.isHosted()) return; if (deployState.getProperties().applicationId().instance().isTester()) return; + // Add endpoints provided by the controller + List<String> hosts = getContainers().stream().map(AbstractService::getHostName).sorted().toList(); List<ApplicationClusterEndpoint> endpoints = new ArrayList<>(); - - List<String> hosts = getContainers().stream() - .map(AbstractService::getHostName) - .sorted() - .toList(); - - Set<ContainerEndpoint> endpointsFromController = deployState.getEndpoints(); - // Add zone-scoped endpoints if not provided by the controller - // TODO(mpolden): Remove this when controller always includes zone-scope endpoints, and config models < 8.230 are gone - if (endpointsFromController.stream().noneMatch(endpoint -> endpoint.scope() == ApplicationClusterEndpoint.Scope.zone)) { - for (String suffix : deployState.getProperties().zoneDnsSuffixes()) { - ApplicationClusterEndpoint.DnsName l4Name = ApplicationClusterEndpoint.DnsName.sharedL4NameFrom( - deployState.zone().system(), - ClusterSpec.Id.from(getName()), - deployState.getProperties().applicationId(), - suffix); - endpoints.add(ApplicationClusterEndpoint.builder() - .zoneScope() - .sharedL4Routing() - .dnsName(l4Name) - .hosts(hosts) - .clusterId(getName()) - .authMethod(ApplicationClusterEndpoint.AuthMethod.mtls) - .build()); - } - } - - // Include all endpoints provided by controller - endpointsFromController.stream() - .filter(ce -> ce.clusterId().equals(getName())) - .forEach(ce -> ce.names().forEach( - name -> endpoints.add(ApplicationClusterEndpoint.builder() - .scope(ce.scope()) - .weight(ce.weight().orElse(1)) // Default to weight=1 if not set - .routingMethod(ce.routingMethod()) - .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) - .hosts(hosts) - .clusterId(getName()) - .authMethod(ce.authMethod()) - .build()) - )); - this.endpoints = List.copyOf(endpoints); + deployState.getEndpoints().stream() + .filter(ce -> ce.clusterId().equals(getName())) + .forEach(ce -> ce.names().forEach( + name -> endpoints.add(ApplicationClusterEndpoint.builder() + .scope(ce.scope()) + .weight(ce.weight().orElse(1)) + .routingMethod(ce.routingMethod()) + .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) + .hosts(hosts) + .clusterId(getName()) + .authMethod(ce.authMethod()) + .build()) + )); + this.endpoints = Collections.unmodifiableList(endpoints); } @Override @@ -299,12 +295,15 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat @Override public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); + var memoryPct = getMemoryPercentage().orElse(null); + int heapsize = memoryPct != null && memoryPct.availableMemoryGb().isPresent() + ? (int) (memoryPct.availableMemoryGb().getAsDouble() * 1024) : 1536; builder.jvm.verbosegc(true) .availableProcessors(0) .compressedClassSpaceSize(0) - .minHeapsize(1536) - .heapsize(1536); - getMemoryPercentage().ifPresent(percentage -> builder.jvm.heapSizeAsPercentageOfPhysicalMemory(percentage)); + .minHeapsize(heapsize) + .heapsize(heapsize); + if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.percentage()); } @Override @@ -373,6 +372,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat @Override public String name() { return getName(); } + public OnnxModelCost.Calculator onnxModelCost() { return onnxModelCost; } + public static class MbusParams { // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) final Double maxConcurrentFactor; @@ -390,4 +391,14 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } } + public static class UserConfiguredUrls { + + private final Set<String> urls = new HashSet<>(); + + public void add(String url) { urls.add(url); } + + public Set<String> all() { return urls; } + + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6bbc24e8739..3d4ec51c8d2 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -62,6 +62,7 @@ import com.yahoo.vespa.model.container.search.ContainerSearch; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.search.SearchCluster; + import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; @@ -71,6 +72,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalDouble; import java.util.Set; import java.util.TreeSet; @@ -142,7 +144,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> private ContainerDocproc containerDocproc; private ContainerDocumentApi containerDocumentApi; private SecretStore secretStore; - private final ContainerThreadpool defaultHandlerThreadpool = new Handler.DefaultHandlerThreadpool(); + private final ContainerThreadpool defaultHandlerThreadpool; private boolean rpcServerEnabled = true; private boolean httpServerEnabled = true; @@ -185,6 +187,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addCommonVespaBundles(); addSimpleComponent(VoidRequestLog.class); addComponent(new DefaultThreadpoolProvider(this, defaultPoolNumThreads)); + defaultHandlerThreadpool = new Handler.DefaultHandlerThreadpool(deployState, null); addComponent(defaultHandlerThreadpool); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); @@ -718,5 +721,11 @@ public abstract class ContainerCluster<CONTAINER extends Container> * Returns the percentage of host physical memory this application has specified for nodes in this cluster, * or empty if this is not specified by the application. */ - public Optional<Integer> getMemoryPercentage() { return Optional.empty(); } + public record JvmMemoryPercentage(int percentage, OptionalDouble availableMemoryGb) { + static JvmMemoryPercentage of(int percentage) { return new JvmMemoryPercentage(percentage, OptionalDouble.empty()); } + static JvmMemoryPercentage of(int percentage, double availableMemoryGb) { + return new JvmMemoryPercentage(percentage, OptionalDouble.of(availableMemoryGb)); + } + } + public Optional<JvmMemoryPercentage> getMemoryPercentage() { return Optional.empty(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 906ef739ef1..1b47f59653e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -45,10 +45,6 @@ public class ContainerModelEvaluation implements private final RankProfileList rankProfileList; private final FileDistributedOnnxModels onnxModels; // For cluster specific ONNX model settings - public ContainerModelEvaluation(ApplicationContainerCluster cluster, RankProfileList rankProfileList) { - this(cluster, rankProfileList, null); - } - public ContainerModelEvaluation(ApplicationContainerCluster cluster, RankProfileList rankProfileList, FileDistributedOnnxModels onnxModels) { this.rankProfileList = Objects.requireNonNull(rankProfileList, "rankProfileList cannot be null"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java index fb4e62f5cd1..4b85c384951 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java @@ -1,16 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.threadpool.ContainerThreadPool; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.container.handler.threadpool.ContainerThreadpoolImpl; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.text.XML; import com.yahoo.vespa.model.container.component.SimpleComponent; import org.w3c.dom.Element; -import java.util.Optional; +import java.util.logging.Level; /** * Component definition for a {@link java.util.concurrent.Executor} using {@link ContainerThreadPool}. @@ -20,18 +21,50 @@ import java.util.Optional; public abstract class ContainerThreadpool extends SimpleComponent implements ContainerThreadpoolConfig.Producer { private final String name; - private final UserOptions userOptions; + private final UserOptions options; - public ContainerThreadpool(String name, UserOptions userOptions) { + record UserOptions(Double max, Double min, Double queue){} + + protected ContainerThreadpool(DeployState ds, String name, Element parent) { super(new ComponentModel( BundleInstantiationSpecification.fromStrings( "threadpool@" + name, ContainerThreadpoolImpl.class.getName(), null))); this.name = name; - this.userOptions = userOptions; + var threadpoolElem = XmlHelper.getOptionalChild(parent, "threadpool").orElse(null); + if (threadpoolElem == null) options = new UserOptions(null, null, null); + else { + // TODO Vespa 9 Remove min-threads, max-threads and queue-size + Double max = null; + Double min = null; + Double queue = null; + var minElem = XmlHelper.getOptionalChild(threadpoolElem, "min-threads").orElse(null); + if (minElem != null) ds.getDeployLogger().logApplicationPackage(Level.WARNING, "For <threadpool>: <min-threads> is deprecated, use <threads> instead"); + var maxElem = XmlHelper.getOptionalChild(threadpoolElem, "max-threads").orElse(null); + if (maxElem != null) ds.getDeployLogger().logApplicationPackage(Level.WARNING, "For <threadpool>: <max-threads> is deprecated, use <threads> with 'boost' instead"); + var queueElem = XmlHelper.getOptionalChild(threadpoolElem, "queue").orElse(null); + var queueSizeElem = XmlHelper.getOptionalChild(threadpoolElem, "queue-size").orElse(null); + if (queueSizeElem != null) ds.getDeployLogger().logApplicationPackage(Level.WARNING, "For <threadpool>: <queue-size> is deprecated, use <queue> instead"); + var threadsElem = XmlHelper.getOptionalChild(threadpoolElem, "threads").orElse(null); + if (threadsElem != null) { + min = parseMultiplier(threadsElem.getTextContent()); + max = threadsElem.hasAttribute("boost") ? parseMultiplier(threadsElem.getAttribute("boost")) : min; + } else if (minElem != null) { + min = parseFixed(minElem.getTextContent()); + } + if (max == null && maxElem != null) { + max = parseFixed(maxElem.getTextContent()); + } + if (queueElem != null) queue = parseMultiplier(queueElem.getTextContent()); + else if (queueSizeElem != null) queue = parseFixed(queueSizeElem.getTextContent()); + options = new UserOptions(max, min, queue); + } } + private static Double parseMultiplier(String text) { return -parseFixed(text); } + private static Double parseFixed(String text) { return Double.parseDouble(text); } + // Must be implemented by subclasses to set values that may be overridden by user options. protected abstract void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder); @@ -40,35 +73,20 @@ public abstract class ContainerThreadpool extends SimpleComponent implements Con setDefaultConfigValues(builder); builder.name(this.name); - if (userOptions != null) { - builder.maxThreads(userOptions.maxThreads); - builder.minThreads(userOptions.minThreads); - builder.queueSize(userOptions.queueSize); + if (options.max() != null) { + int max = (int) Math.round(options.max()); + if (options.max() != 0 && max == 0) max = options.max() > 0 ? 1 : -1; + builder.maxThreads(max); } - } - - public static class UserOptions { - private final int maxThreads; - private final int minThreads; - private final int queueSize; - - private UserOptions(int maxThreads, int minThreads, int queueSize) { - this.maxThreads = maxThreads; - this.minThreads = minThreads; - this.queueSize = queueSize; - } - - public static Optional<UserOptions> fromXml(Element xml) { - Element element = XML.getChild(xml, "threadpool"); - if (element == null) return Optional.empty(); - return Optional.of(new UserOptions( - intOption(element, "max-threads"), - intOption(element, "min-threads"), - intOption(element, "queue-size"))); + if (options.min() != null) { + int min = (int) Math.round(options.min()); + if (options.min() != 0 && min == 0) min = options.min() > 0 ? 1 : -1; + builder.minThreads(min); } - - private static int intOption(Element element, String name) { - return Integer.parseInt(XML.getChild(element, name).getTextContent()); + if (options.queue() != null) { + int queue = (int) Math.round(options.queue()); + if (options.queue() != 0 && queue == 0) queue = options.queue() > 0 ? 1 : -1; + builder.queueSize(queue); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java index 205848e1b67..d02b7d0de5f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java @@ -5,10 +5,9 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.embedding.BertBaseEmbedderConfig; -import com.yahoo.vespa.model.container.xml.ModelIdResolver; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.w3c.dom.Element; -import static com.yahoo.text.XML.getChild; import static com.yahoo.text.XML.getChildValue; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; @@ -17,8 +16,8 @@ import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATI */ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConfig.Producer { - private final ModelReference model; - private final ModelReference vocab; + private final ModelReference modelRef; + private final ModelReference vocabRef; private final Integer maxTokens; private final String transformerInputIds; private final String transformerAttentionMask; @@ -33,10 +32,11 @@ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConf private final Integer onnxGpuDevice; - public BertEmbedder(Element xml, DeployState state) { + public BertEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.BertBaseEmbedder", INTEGRATION_BUNDLE_NAME, xml); - model = ModelIdResolver.resolveToModelReference(getChild(xml, "transformer-model"), state); - vocab = ModelIdResolver.resolveToModelReference(getChild(xml, "tokenizer-vocab"), state); + var model = Model.fromXml(state, xml, "transformer-model").orElseThrow(); + modelRef = model.modelReference(); + vocabRef = Model.fromXml(state, xml, "tokenizer-vocab").orElseThrow().modelReference(); maxTokens = getChildValue(xml, "max-tokens").map(Integer::parseInt).orElse(null); transformerInputIds = getChildValue(xml, "transformer-input-ids").orElse(null); transformerAttentionMask = getChildValue(xml, "transformer-attention-mask").orElse(null); @@ -49,11 +49,12 @@ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConf onnxInteropThreads = getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt).orElse(null); onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); + model.registerOnnxModelCost(cluster); } @Override public void getConfig(BertBaseEmbedderConfig.Builder b) { - b.transformerModel(model).tokenizerVocab(vocab); + b.transformerModel(modelRef).tokenizerVocab(vocabRef); if (maxTokens != null) b.transformerMaxTokens(maxTokens); if (transformerInputIds != null) b.transformerInputIds(transformerInputIds); if (transformerAttentionMask != null) b.transformerAttentionMask(transformerAttentionMask); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java index c0fdfe3dc64..66e3b1c9dfd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java @@ -5,13 +5,9 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.embedding.ColBertEmbedderConfig; -import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; -import com.yahoo.vespa.model.container.xml.ModelIdResolver; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.w3c.dom.Element; -import java.util.Optional; - -import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChild; import static com.yahoo.text.XML.getChildValue; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; @@ -20,8 +16,8 @@ import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATI * @author bergum */ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderConfig.Producer { - private final ModelReference model; - private final ModelReference vocab; + private final ModelReference modelRef; + private final ModelReference vocabRef; private final Integer maxQueryTokens; @@ -40,13 +36,13 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo private final Integer onnxIntraopThreads; private final Integer onnxGpuDevice; - public ColBertEmbedder(Element xml, DeployState state) { + public ColBertEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.ColBertEmbedder", INTEGRATION_BUNDLE_NAME, xml); - var transformerModelElem = getOptionalChild(xml, "transformer-model").orElseThrow(); - model = ModelIdResolver.resolveToModelReference(transformerModelElem, state); - vocab = getOptionalChild(xml, "tokenizer-model") - .map(elem -> ModelIdResolver.resolveToModelReference(elem, state)) - .orElseGet(() -> resolveDefaultVocab(transformerModelElem, state)); + var model = Model.fromXml(state, xml, "transformer-model").orElseThrow(); + modelRef = model.modelReference(); + vocabRef = Model.fromXml(state, xml, "tokenizer-model") + .map(Model::modelReference) + .orElseGet(() -> resolveDefaultVocab(model, state)); maxTokens = getChildValue(xml, "max-tokens").map(Integer::parseInt).orElse(null); maxQueryTokens = getChildValue(xml, "max-query-tokens").map(Integer::parseInt).orElse(null); maxDocumentTokens = getChildValue(xml, "max-document-tokens").map(Integer::parseInt).orElse(null); @@ -60,21 +56,20 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo onnxInteropThreads = getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt).orElse(null); onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); - + model.registerOnnxModelCost(cluster); } - private static ModelReference resolveDefaultVocab(Element model, DeployState state) { - if (state.isHosted() && model.hasAttribute("model-id")) { - var implicitVocabId = model.getAttribute("model-id") + "-vocab"; - return ModelIdResolver.resolveToModelReference( - "tokenizer-model", Optional.of(implicitVocabId), Optional.empty(), Optional.empty(), state); + private static ModelReference resolveDefaultVocab(Model model, DeployState state) { + var modelId = model.modelId().orElse(null); + if (state.isHosted() && modelId != null) { + return Model.fromParams(state, model.name(), modelId + "-vocab", null, null).modelReference(); } throw new IllegalArgumentException("'tokenizer-model' must be specified"); } @Override public void getConfig(ColBertEmbedderConfig.Builder b) { - b.transformerModel(model).tokenizerPath(vocab); + b.transformerModel(modelRef).tokenizerPath(vocabRef); if (maxTokens != null) b.transformerMaxTokens(maxTokens); if (transformerInputIds != null) b.transformerInputIds(transformerInputIds); if (transformerAttentionMask != null) b.transformerAttentionMask(transformerAttentionMask); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index 31031aa5bf2..969db6553e6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -1,9 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.component; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerThreadpool; +import org.w3c.dom.Element; import java.util.ArrayList; import java.util.Arrays; @@ -76,8 +78,8 @@ public class Handler extends Component<Component<?, ?>, ComponentModel> { */ public static class DefaultHandlerThreadpool extends ContainerThreadpool { - public DefaultHandlerThreadpool() { - super("default-handler-common", null); + public DefaultHandlerThreadpool(DeployState ds, Element options) { + super(ds, "default-handler-common", options); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java index f4017339699..af47bee137a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java @@ -5,12 +5,9 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; -import com.yahoo.vespa.model.container.xml.ModelIdResolver; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.w3c.dom.Element; -import java.util.Optional; - -import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChild; import static com.yahoo.text.XML.getChildValue; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; @@ -19,8 +16,8 @@ import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATI * @author bjorncs */ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEmbedderConfig.Producer { - private final ModelReference model; - private final ModelReference vocab; + private final ModelReference modelRef; + private final ModelReference vocabRef; private final Integer maxTokens; private final String transformerInputIds; private final String transformerAttentionMask; @@ -33,13 +30,13 @@ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEm private final Integer onnxGpuDevice; private final String poolingStrategy; - public HuggingFaceEmbedder(Element xml, DeployState state) { + public HuggingFaceEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.huggingface.HuggingFaceEmbedder", INTEGRATION_BUNDLE_NAME, xml); - var transformerModelElem = getOptionalChild(xml, "transformer-model").orElseThrow(); - model = ModelIdResolver.resolveToModelReference(transformerModelElem, state); - vocab = getOptionalChild(xml, "tokenizer-model") - .map(elem -> ModelIdResolver.resolveToModelReference(elem, state)) - .orElseGet(() -> resolveDefaultVocab(transformerModelElem, state)); + var model = Model.fromXml(state, xml, "transformer-model").orElseThrow(); + modelRef = model.modelReference(); + vocabRef = Model.fromXml(state, xml, "tokenizer-model") + .map(Model::modelReference) + .orElseGet(() -> resolveDefaultVocab(model, state)); maxTokens = getChildValue(xml, "max-tokens").map(Integer::parseInt).orElse(null); transformerInputIds = getChildValue(xml, "transformer-input-ids").orElse(null); transformerAttentionMask = getChildValue(xml, "transformer-attention-mask").orElse(null); @@ -51,20 +48,20 @@ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEm onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); poolingStrategy = getChildValue(xml, "pooling-strategy").orElse(null); + model.registerOnnxModelCost(cluster); } - private static ModelReference resolveDefaultVocab(Element model, DeployState state) { - if (state.isHosted() && model.hasAttribute("model-id")) { - var implicitVocabId = model.getAttribute("model-id") + "-vocab"; - return ModelIdResolver.resolveToModelReference( - "tokenizer-model", Optional.of(implicitVocabId), Optional.empty(), Optional.empty(), state); + private static ModelReference resolveDefaultVocab(Model model, DeployState state) { + var modelId = model.modelId().orElse(null); + if (state.isHosted() && modelId != null) { + return Model.fromParams(state, model.name(), modelId + "-vocab", null, null).modelReference(); } throw new IllegalArgumentException("'tokenizer-model' must be specified"); } @Override public void getConfig(HuggingFaceEmbedderConfig.Builder b) { - b.transformerModel(model).tokenizerPath(vocab); + b.transformerModel(modelRef).tokenizerPath(vocabRef); if (maxTokens != null) b.transformerMaxTokens(maxTokens); if (transformerInputIds != null) b.transformerInputIds(transformerInputIds); if (transformerAttentionMask != null) b.transformerAttentionMask(transformerAttentionMask); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java index 0bf5491e872..e9ac93caa68 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceTokenizer.java @@ -7,7 +7,6 @@ import com.yahoo.language.huggingface.config.HuggingFaceTokenizerConfig; import com.yahoo.language.huggingface.config.HuggingFaceTokenizerConfig.Padding; import com.yahoo.language.huggingface.config.HuggingFaceTokenizerConfig.Truncation; import com.yahoo.text.XML; -import com.yahoo.vespa.model.container.xml.ModelIdResolver; import org.w3c.dom.Element; import java.util.Map; @@ -26,7 +25,7 @@ public class HuggingFaceTokenizer extends TypedComponent implements HuggingFaceT super("com.yahoo.language.huggingface.HuggingFaceTokenizer", LINGUISTICS_BUNDLE_NAME, xml); for (Element element : XML.getChildren(xml, "model")) { var lang = element.hasAttribute("language") ? element.getAttribute("language") : "unknown"; - langToModel.put(lang, ModelIdResolver.resolveToModelReference(element, state)); + langToModel.put(lang, Model.fromXml(state, element).modelReference()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java new file mode 100644 index 00000000000..76d93c38aee --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.model.container.component; + +import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.path.Path; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.xml.ModelIdResolver; +import org.w3c.dom.Element; + +import java.net.URI; +import java.util.Objects; +import java.util.Optional; + +/** + * Represents a model, e.g ONNX model for an embedder. + * + * @author bjorncs + */ +class Model { + private final String paramName; + private final String modelId; + private final URI url; + private final ApplicationFile file; + private final ModelReference ref; + + private Model(DeployState ds, String paramName, String modelId, URI url, Path file) { + this.paramName = Objects.requireNonNull(paramName); + if (modelId == null && url == null && file == null) + throw new IllegalArgumentException("At least one of 'model-id', 'url' or 'path' must be specified"); + this.modelId = modelId; + this.url = url; + this.file = file != null ? ds.getApplicationPackage().getFile(file) : null; + this.ref = ModelIdResolver.resolveToModelReference( + paramName, Optional.ofNullable(modelId), Optional.ofNullable(url).map(URI::toString), + Optional.ofNullable(file).map(Path::toString), ds); + } + + static Model fromParams(DeployState ds, String paramName, String modelId, URI url, Path file) { + return new Model(ds, paramName, modelId, url, file); + } + + static Optional<Model> fromXml(DeployState ds, Element parent, String name) { + return XmlHelper.getOptionalChild(parent, name).map(e -> fromXml(ds, e)); + } + + static Model fromXml(DeployState ds, Element model) { + var modelId = XmlHelper.getOptionalAttribute(model, "model-id").orElse(null); + var url = XmlHelper.getOptionalAttribute(model, "url").map(URI::create).orElse(null); + var path = XmlHelper.getOptionalAttribute(model, "path").map(Path::fromString).orElse(null); + return new Model(ds, model.getTagName(), modelId, url, path); + } + + void registerOnnxModelCost(ApplicationContainerCluster c) { + var resolvedUrl = resolvedUrl().orElse(null); + if (file != null) c.onnxModelCost().registerModel(file); + else if (resolvedUrl != null) c.onnxModelCost().registerModel(resolvedUrl); + } + + String name() { return paramName; } + Optional<String> modelId() { return Optional.ofNullable(modelId); } + Optional<URI> url() { return Optional.ofNullable(url); } + Optional<URI> resolvedUrl() { return ref.url().map(u -> URI.create(u.value())); } + Optional<ApplicationFile> file() { return Optional.ofNullable(file); } + ModelReference modelReference() { return ref; } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponent.java index c0431d01784..2354298779d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponent.java @@ -29,8 +29,6 @@ public class ChainedComponent<T extends ChainedComponentModel> extends Component private ComponentId namespace() { var owner = getParent().getParent(); - return (owner instanceof Chain) ? - ((Chain) owner).getGlobalComponentId() : - null; + return (owner instanceof Chain<?> chain) ? chain.getGlobalComponentId() : null; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index f0296d49472..3261d454b4f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; @@ -56,12 +57,14 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> private QueryProfiles queryProfiles; private SemanticRules semanticRules; private PageTemplates pageTemplates; + private ApplicationPackage app; public ContainerSearch(DeployState deployState, ApplicationContainerCluster cluster, SearchChains chains) { super(chains); this.globalPhase = deployState.featureFlags().enableGlobalPhase(); this.useReconfigurableDispatcher = deployState.featureFlags().useReconfigurableDispatcher(); this.schemasWithGlobalPhase = getSchemasWithGlobalPhase(deployState); + this.app = deployState.getApplicationPackage(); this.owningCluster = cluster; owningCluster.addComponent(Component.fromClassAndBundle(CompiledQueryProfileRegistry.class, SEARCH_AND_DOCPROC_BUNDLE)); @@ -96,6 +99,9 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> if ( ! schemasWithGlobalPhase.contains(documentDb.getSchemaName())) continue; var factory = new RankProfilesEvaluatorComponent(documentDb); if ( ! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) { + var onnxModels = documentDb.getDerivedConfiguration().getRankProfileList().getOnnxModels(); + onnxModels.asMap().forEach( + (__, model) -> owningCluster.onnxModelCost().registerModel(app.getFile(model.getFilePath()))); owningCluster.addComponent(factory); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 35b0213bf59..1874b5fa19a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -39,6 +39,8 @@ import com.yahoo.container.jdisc.DataplaneProxyService; import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudTokenDataPlaneFilterConfig; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudTokenDataPlaneFilterConfig.Builder; import com.yahoo.jdisc.http.server.jetty.DataplaneProxyCredentials; import com.yahoo.jdisc.http.server.jetty.VoidRequestLog; import com.yahoo.osgi.provider.model.ComponentModel; @@ -68,7 +70,6 @@ import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.ContainerModelEvaluation; -import com.yahoo.vespa.model.container.ContainerThreadpool; import com.yahoo.vespa.model.container.DataplaneProxy; import com.yahoo.vespa.model.container.IdentityProvider; import com.yahoo.vespa.model.container.PlatformBundles; @@ -240,10 +241,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addParameterStoreValidationHandler(ApplicationContainerCluster cluster, DeployState deployState) { + if ( ! deployState.isHosted()) return; // Always add platform bundle. Cannot be controlled by a feature flag as platform bundle cannot change. - if(deployState.isHosted()) { - cluster.addPlatformBundle(PlatformBundles.absoluteBundlePath("jdisc-cloud-aws")); - } + cluster.addPlatformBundle(PlatformBundles.absoluteBundlePath("jdisc-cloud-aws")); if (deployState.zone().system().isPublic()) { BindingPattern bindingPattern = SystemBindingPattern.fromHttpPath("/validate-secret-store"); Handler handler = new Handler( @@ -459,7 +459,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private static void addCloudDataPlaneFilter(DeployState deployState, ApplicationContainerCluster cluster) { if (!deployState.isHosted() || !deployState.zone().system().isPublic()) return; - var dataplanePort = getMtlsDataplanePort(deployState, cluster); + var dataplanePort = getMtlsDataplanePort(deployState); // Setup secure filter chain var secureChain = new HttpFilterChain("cloud-data-plane-secure", HttpFilterChain.Type.SYSTEM); secureChain.addInnerComponent(new CloudDataPlaneFilter(cluster, deployState)); @@ -594,7 +594,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { String serverName = server.getComponentId().getName(); // If the deployment contains certificate/private key reference, setup TLS port - var builder = HostedSslConnectorFactory.builder(serverName, getMtlsDataplanePort(state, cluster)) + var builder = HostedSslConnectorFactory.builder(serverName, getMtlsDataplanePort(state)) .proxyProtocol(true, state.getProperties().featureFlags().enableProxyProtocolMixedMode()) .tlsCiphersOverride(state.getProperties().tlsCiphersOverride()) .endpointConnectionTtl(state.getProperties().endpointConnectionTtl()); @@ -627,19 +627,19 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addCloudTokenSupport(DeployState state, ApplicationContainerCluster cluster) { var server = cluster.getHttp().getHttpServer().get(); - if (!enableTokenSupport(state, cluster)) return; + if (!enableTokenSupport(state)) return; Set<String> tokenEndpoints = tokenEndpoints(state).stream() .map(ContainerEndpoint::names) .flatMap(Collection::stream) .collect(Collectors.toSet()); var endpointCert = state.endpointCertificateSecrets().orElseThrow(); - int tokenPort = getTokenDataplanePort(state, cluster).orElseThrow(); + int tokenPort = getTokenDataplanePort(state).orElseThrow(); // Set up component to generate proxy cert if token support is enabled cluster.addSimpleComponent(DataplaneProxyCredentials.class); cluster.addSimpleComponent(DataplaneProxyService.class); var dataplaneProxy = new DataplaneProxy( - getMtlsDataplanePort(state, cluster), + getMtlsDataplanePort(state), tokenPort, endpointCert.certificate(), endpointCert.key(), @@ -659,13 +659,24 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Setup token filter chain var tokenChain = new HttpFilterChain("cloud-token-data-plane-secure", HttpFilterChain.Type.SYSTEM); - tokenChain.addInnerComponent(new CloudTokenDataPlaneFilter(cluster, state)); + var tokenFilter = new CloudTokenDataPlaneFilter(cluster, state); + tokenChain.addInnerComponent(tokenFilter); cluster.getHttp().getFilterChains().add(tokenChain); // Set as default filter for token port cluster.getHttp().getHttpServer().orElseThrow().getConnectorFactories().stream() .filter(c -> c.getListenPort() == tokenPort).findAny().orElseThrow() .setDefaultRequestFilterChain(tokenChain.getComponentId()); + + // Set up handler that tells what fingerprints are known to the container + class CloudTokenDataPlaneHandler extends Handler implements CloudTokenDataPlaneFilterConfig.Producer { + CloudTokenDataPlaneHandler() { + super(new ComponentModel("com.yahoo.jdisc.http.filter.security.cloud.CloudTokenDataPlaneHandler", null, "jdisc-security-filters", null)); + addServerBindings(SystemBindingPattern.fromHttpPortAndPath(Defaults.getDefaults().vespaWebServicePort(), "/data-plane-tokens/v1")); + } + @Override public void getConfig(Builder builder) { tokenFilter.getConfig(builder); } + } + cluster.addComponent(new CloudTokenDataPlaneHandler()); } // Returns the client certificates of the clients defined for an application cluster @@ -710,7 +721,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private Http buildHttp(DeployState deployState, ApplicationContainerCluster cluster, Element httpElement, ConfigModelContext context) { - Http http = new HttpBuilder(portBindingOverride(deployState, context, cluster)).build(deployState, cluster, httpElement); + Http http = new HttpBuilder(portBindingOverride(deployState, context)).build(deployState, cluster, httpElement); if (networking == Networking.disable) http.removeAllServers(); @@ -778,6 +789,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { !container.getHostResource().realResources().gpuResources().isZero()); onnxModel.setGpuDevice(gpuDevice, hasGpu); } + cluster.onnxModelCost().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath())); } cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles, models)); @@ -815,7 +827,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addSearchAndDocprocBundles(); addIncludes(processingElement); cluster.setProcessingChains(new DomProcessingBuilder(null).build(deployState, cluster, processingElement), - serverBindings(deployState, context, processingElement, ProcessingChains.defaultBindings, cluster).toArray(BindingPattern[]::new)); + serverBindings(deployState, context, processingElement, ProcessingChains.defaultBindings).toArray(BindingPattern[]::new)); validateAndAddConfiguredComponents(deployState, cluster, processingElement, "renderer", ContainerModelBuilder::validateRendererElement); } @@ -840,7 +852,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addUserHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { for (Element component: XML.getChildren(spec, "handler")) { cluster.addComponent( - new DomHandlerBuilder(cluster, portBindingOverride(deployState, context, cluster)).build(deployState, cluster, component)); + new DomHandlerBuilder(cluster, portBindingOverride(deployState, context)).build(deployState, cluster, component)); } } @@ -1128,28 +1140,28 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addSearchHandler(DeployState deployState, ApplicationContainerCluster cluster, Element searchElement, ConfigModelContext context) { var bindingPatterns = List.<BindingPattern>of(SearchHandler.DEFAULT_BINDING); if (isHostedTenantApplication(context)) { - bindingPatterns = SearchHandler.bindingPattern(getDataplanePorts(deployState, cluster)); + bindingPatterns = SearchHandler.bindingPattern(getDataplanePorts(deployState)); } - SearchHandler searchHandler = new SearchHandler(cluster, - serverBindings(deployState, context, searchElement, bindingPatterns, cluster), - ContainerThreadpool.UserOptions.fromXml(searchElement).orElse(null)); + SearchHandler searchHandler = new SearchHandler(deployState, cluster, + serverBindings(deployState, context, searchElement, bindingPatterns), + searchElement); cluster.addComponent(searchHandler); // Add as child to SearchHandler to get the correct chains config. searchHandler.addComponent(Component.fromClassAndBundle(SearchHandler.EXECUTION_FACTORY, PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE)); } - private List<BindingPattern> serverBindings(DeployState deployState, ConfigModelContext context, Element searchElement, Collection<BindingPattern> defaultBindings, ApplicationContainerCluster cluster) { + private List<BindingPattern> serverBindings(DeployState deployState, ConfigModelContext context, Element searchElement, Collection<BindingPattern> defaultBindings) { List<Element> bindings = XML.getChildren(searchElement, "binding"); if (bindings.isEmpty()) return List.copyOf(defaultBindings); - return toBindingList(deployState, context, bindings, cluster); + return toBindingList(deployState, context, bindings); } - private List<BindingPattern> toBindingList(DeployState deployState, ConfigModelContext context, List<Element> bindingElements, ApplicationContainerCluster cluster) { + private List<BindingPattern> toBindingList(DeployState deployState, ConfigModelContext context, List<Element> bindingElements) { List<BindingPattern> result = new ArrayList<>(); - var portOverride = isHostedTenantApplication(context) ? getDataplanePorts(deployState, cluster) : Set.<Integer>of(); + var portOverride = isHostedTenantApplication(context) ? getDataplanePorts(deployState) : Set.<Integer>of(); for (Element element: bindingElements) { String text = element.getTextContent().trim(); if (!text.isEmpty()) @@ -1173,13 +1185,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ContainerDocumentApi.HandlerOptions documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement); Element ignoreUndefinedFields = XML.getChild(documentApiElement, "ignore-undefined-fields"); - return new ContainerDocumentApi(cluster, documentApiOptions, - "true".equals(XML.getValue(ignoreUndefinedFields)), portBindingOverride(deployState, context, cluster)); + return new ContainerDocumentApi(deployState, cluster, documentApiOptions, + "true".equals(XML.getValue(ignoreUndefinedFields)), portBindingOverride(deployState, context)); } - private Set<Integer> portBindingOverride(DeployState deployState, ConfigModelContext context, ApplicationContainerCluster cluster) { + private Set<Integer> portBindingOverride(DeployState deployState, ConfigModelContext context) { return isHostedTenantApplication(context) - ? getDataplanePorts(deployState, cluster) + ? getDataplanePorts(deployState) : Set.<Integer>of(); } @@ -1438,18 +1450,18 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } - private static Set<Integer> getDataplanePorts(DeployState ds, ApplicationContainerCluster cluster) { - var tokenPort = getTokenDataplanePort(ds, cluster); - var mtlsPort = getMtlsDataplanePort(ds, cluster); + private static Set<Integer> getDataplanePorts(DeployState ds) { + var tokenPort = getTokenDataplanePort(ds); + var mtlsPort = getMtlsDataplanePort(ds); return tokenPort.isPresent() ? Set.of(mtlsPort, tokenPort.getAsInt()) : Set.of(mtlsPort); } - private static int getMtlsDataplanePort(DeployState ds, ApplicationContainerCluster cluster) { - return enableTokenSupport(ds, cluster) ? 8443 : 4443; + private static int getMtlsDataplanePort(DeployState ds) { + return enableTokenSupport(ds) ? 8443 : 4443; } - private static OptionalInt getTokenDataplanePort(DeployState ds, ApplicationContainerCluster cluster) { - return enableTokenSupport(ds, cluster) ? OptionalInt.of(8444) : OptionalInt.empty(); + private static OptionalInt getTokenDataplanePort(DeployState ds) { + return enableTokenSupport(ds) ? OptionalInt.of(8444) : OptionalInt.empty(); } private static Set<ContainerEndpoint> tokenEndpoints(DeployState deployState) { @@ -1458,7 +1470,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .collect(Collectors.toSet()); } - private static boolean enableTokenSupport(DeployState state, ApplicationContainerCluster cluster) { + private static boolean enableTokenSupport(DeployState state) { Set<ContainerEndpoint> tokenEndpoints = tokenEndpoints(state); return state.isHosted() && state.zone().system().isPublic() && ! tokenEndpoints.isEmpty(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java index bb1d0af1db9..cdbe62720b9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.text.XML; import com.yahoo.vespa.model.clients.ContainerDocumentApi; -import com.yahoo.vespa.model.container.ContainerThreadpool; import org.w3c.dom.Element; import java.util.ArrayList; @@ -19,13 +18,7 @@ public class DocumentApiOptionsBuilder { private static final Logger log = Logger.getLogger(DocumentApiOptionsBuilder.class.getName()); public static ContainerDocumentApi.HandlerOptions build(Element spec) { - return new ContainerDocumentApi.HandlerOptions(getBindings(spec), threadpoolOptions(spec, "http-client-api")); - } - - private static ContainerThreadpool.UserOptions threadpoolOptions(Element spec, String elementName) { - Element element = XML.getChild(spec, elementName); - if (element == null) return null; - return ContainerThreadpool.UserOptions.fromXml(element).orElse(null); + return new ContainerDocumentApi.HandlerOptions(getBindings(spec), XML.getChild(spec, "http-client-api")); } private static List<String> getBindings(Element spec) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java index be3ca0b8aa9..14216dd8855 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.ModelReference; import com.yahoo.config.UrlReference; -import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.text.XML; import org.w3c.dom.Element; @@ -88,13 +87,6 @@ public class ModelIdResolver { } } - - public static ModelReference resolveToModelReference(Element elem, DeployState state) { - return resolveToModelReference( - elem.getTagName(), XmlHelper.getOptionalAttribute(elem, "model-id"), - XmlHelper.getOptionalAttribute(elem, "url"), XmlHelper.getOptionalAttribute(elem, "path"), state); - } - public static ModelReference resolveToModelReference( String paramName, Optional<String> id, Optional<String> url, Optional<String> path, DeployState state) { if (id.isEmpty()) return createModelReference(Optional.empty(), url, path, state); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java index 6cfef153fee..3cd296c1469 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; @@ -9,6 +10,7 @@ import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import org.w3c.dom.Element; import java.util.Collection; import java.util.List; @@ -30,10 +32,11 @@ class SearchHandler extends ProcessingHandler<SearchChains> { static final BundleInstantiationSpecification HANDLER_SPEC = fromSearchAndDocproc(HANDLER_CLASSNAME); static final BindingPattern DEFAULT_BINDING = SystemBindingPattern.fromHttpPath("/search/*"); - SearchHandler(ApplicationContainerCluster cluster, + SearchHandler(DeployState ds, + ApplicationContainerCluster cluster, List<BindingPattern> bindings, - ContainerThreadpool.UserOptions threadpoolOptions) { - super(cluster.getSearchChains(), HANDLER_SPEC, new Threadpool(threadpoolOptions)); + Element threadpoolOptions) { + super(cluster.getSearchChains(), HANDLER_SPEC, new Threadpool(ds, threadpoolOptions)); bindings.forEach(this::addServerBindings); } @@ -46,8 +49,8 @@ class SearchHandler extends ProcessingHandler<SearchChains> { private static class Threadpool extends ContainerThreadpool { - Threadpool(UserOptions options) { - super("search-handler", options); + Threadpool(DeployState ds, Element options) { + super(ds, "search-handler", options); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index bb72eda7d04..d18309ef0af 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -256,7 +256,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem for (ContainerModel containerModel : containers) { Optional<String> hostClusterId = containerModel.getCluster().getHostClusterId(); if (hostClusterId.isPresent() && hostClusterId.get().equals(clusterId) && containerModel.getCluster().getMemoryPercentage().isPresent()) { - return containerModel.getCluster().getMemoryPercentage().get() * 0.01; + return containerModel.getCluster().getMemoryPercentage().get().percentage() * 0.01; } } return 0.0; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java index 8bed5e64bf5..8352a011b88 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java @@ -5,12 +5,14 @@ import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; + import com.yahoo.yolean.Exceptions; import java.io.File; @@ -21,19 +23,28 @@ import java.util.Map; import java.util.Optional; import java.util.logging.Level; +import static com.yahoo.vespa.model.container.ApplicationContainerCluster.UserConfiguredUrls; + /** * Utility methods for registering file distribution of files/paths/urls/models defined by the user. * * @author gjoranv + * @author hmusum */ public class UserConfiguredFiles implements Serializable { private final FileRegistry fileRegistry; private final DeployLogger logger; + private final UserConfiguredUrls userConfiguredUrls; + private final String unknownConfigDefinition; - public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger) { + public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger, + ModelContext.FeatureFlags featureFlags, + UserConfiguredUrls userConfiguredUrls) { this.fileRegistry = fileRegistry; this.logger = logger; + this.userConfiguredUrls = userConfiguredUrls; + this.unknownConfigDefinition = featureFlags.unknownConfigDefinition(); } /** @@ -47,7 +58,7 @@ public class UserConfiguredFiles implements Serializable { try { register(builder, registeredFiles, key); } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Unable to register file specified in services.xml for config '" + key + "': " + + throw new IllegalArgumentException("Invalid config in services.xml for '" + key + "': " + Exceptions.toMessageString(e)); } } @@ -56,9 +67,12 @@ public class UserConfiguredFiles implements Serializable { private void register(ConfigPayloadBuilder builder, Map<Path, FileReference> registeredFiles, ConfigDefinitionKey key) { ConfigDefinition configDefinition = builder.getConfigDefinition(); if (configDefinition == null) { - // TODO: throw new IllegalArgumentException("Unable to find config definition for " + builder); - logger.logApplicationPackage(Level.INFO, "Unable to find config definition " + key + - ". Will not register files for file distribution for this config"); + String message = "Unable to find config definition " + key + ". Will not register files for file distribution for this config"; + switch (unknownConfigDefinition) { + case "log" -> logger.logApplicationPackage(Level.INFO, message); + case "warning" -> logger.logApplicationPackage(Level.WARNING, message); + case "fail" -> throw new IllegalArgumentException("Unable to find config definition for " + key); + } return; } @@ -113,8 +127,7 @@ public class UserConfiguredFiles implements Serializable { ConfigPayloadBuilder fileEntry = builder.getObject(name); if (isEmptyOptionalPath(entry, fileEntry)) continue; if (fileEntry.getValue() == null || fileEntry.getValue().equals(".")) - throw new IllegalArgumentException("Unable to register file for field '" + name + - "': Invalid config value '" + fileEntry.getValue() + "'"); + throw new IllegalArgumentException("Invalid config value '" + fileEntry.getValue() + "' for field '" + name); registerFileEntry(fileEntry, registeredFiles, isModelType); } } @@ -133,7 +146,10 @@ public class UserConfiguredFiles implements Serializable { Path path; if (isModelType) { var modelReference = ModelReference.valueOf(builder.getValue()); - if (modelReference.path().isEmpty()) return; + if (modelReference.path().isEmpty()) { + modelReference.url().ifPresent(url -> userConfiguredUrls.add(url.value())); + return; + } path = Path.fromString(modelReference.path().get().value()); } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java index 7c86267c1b6..39a8e16fad5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java @@ -29,6 +29,7 @@ import java.util.Map; public class OnnxModelProbe { private static final String binary = "vespa-analyze-onnx-model"; + private static final ObjectMapper jsonParser = new ObjectMapper(); static TensorType probeModel(ApplicationPackage app, Path modelPath, String outputName, Map<String, TensorType> inputTypes) { TensorType outputType = TensorType.empty; @@ -41,8 +42,9 @@ public class OnnxModelProbe { // Otherwise, run vespa-analyze-onnx-model if the model is available if (outputType.equals(TensorType.empty) && app.getFile(modelPath).exists()) { String jsonInput = createJsonInput(app.getFileReference(modelPath).getAbsolutePath(), inputTypes); - String jsonOutput = callVespaAnalyzeOnnxModel(jsonInput); + var jsonOutput = callVespaAnalyzeOnnxModel(jsonInput); outputType = outputTypeFromJson(jsonOutput, outputName); + writeMemoryStats(app, modelPath, MemoryStats.fromJson(jsonOutput)); if ( ! outputType.equals(TensorType.empty)) { writeProbedOutputType(app, modelPath, contextKey, outputType); } @@ -53,6 +55,16 @@ public class OnnxModelProbe { return outputType; } + private static void writeMemoryStats(ApplicationPackage app, Path modelPath, MemoryStats memoryStats) throws IOException { + String path = app.getFileReference(memoryStatsPath(modelPath)).getAbsolutePath(); + IOUtils.writeFile(path, memoryStats.toJson().toPrettyString(), false); + } + + private static Path memoryStatsPath(Path modelPath) { + var fileName = OnnxModelInfo.asValidIdentifier(modelPath.getRelative()) + ".memory_stats"; + return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(fileName); + } + private static String createContextKey(String onnxName, Map<String, TensorType> inputTypes) { StringBuilder key = new StringBuilder().append(onnxName).append(":"); inputTypes.entrySet().stream().sorted(Map.Entry.comparingByKey()) @@ -95,9 +107,7 @@ public class OnnxModelProbe { return TensorType.empty; } - private static TensorType outputTypeFromJson(String json, String outputName) throws IOException { - ObjectMapper m = new ObjectMapper(); - JsonNode root = m.readTree(json); + private static TensorType outputTypeFromJson(JsonNode root, String outputName) throws IOException { if ( ! root.isObject() || ! root.has("outputs")) { return TensorType.empty; } @@ -123,7 +133,7 @@ public class OnnxModelProbe { return out.toString(); } - private static String callVespaAnalyzeOnnxModel(String jsonInput) throws IOException, InterruptedException { + private static JsonNode callVespaAnalyzeOnnxModel(String jsonInput) throws IOException, InterruptedException { StringBuilder output = new StringBuilder(); ProcessBuilder processBuilder = new ProcessBuilder(binary, "--probe-types"); @@ -148,7 +158,16 @@ public class OnnxModelProbe { throw new IllegalArgumentException("Error from '" + binary + "'. Return code: " + returnCode + ". " + "Output: '" + output + "'"); } - return output.toString(); + return jsonParser.readTree(output.toString()); + } + + public record MemoryStats(long vmSize, long vmRss) { + static MemoryStats fromJson(JsonNode json) { + return new MemoryStats(json.get("vm_size").asLong(), json.get("vm_rss").asLong()); + } + JsonNode toJson() { + return jsonParser.createObjectNode().put("vm_size", vmSize).put("vm_rss", vmRss); + } } } diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 74f6b5b003c..c10b5a66e06 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -126,9 +126,15 @@ SslProvider = element ssl-provider { } Threadpool = element threadpool { - element max-threads { xsd:nonNegativeInteger } & - element min-threads { xsd:nonNegativeInteger } & - element queue-size { xsd:nonNegativeInteger } + (( + # TODO Vespa 9 Remove max-threads / min-threads / queue-size + element max-threads { xsd:nonNegativeInteger } & + element min-threads { xsd:nonNegativeInteger } & + element queue-size { xsd:nonNegativeInteger } + )|( + element threads { xsd:double { minExclusive = "0.0" } & attribute boost { xsd:double { minExclusive = "0.0" } }? }? & + element queue { xsd:double { minInclusive = "0.0" } }? + )) } Clients = element clients { diff --git a/config-model/src/test/derived/ngram/chunk.sd b/config-model/src/test/derived/ngram/chunk.sd new file mode 100644 index 00000000000..7c2a7465327 --- /dev/null +++ b/config-model/src/test/derived/ngram/chunk.sd @@ -0,0 +1,20 @@ +schema chunk { + + document chunk { + field content type string { + indexing: summary | index + match { + gram + gram-size: 3 + } + } + } + + document-summary content-summary inherits default { + summary content_dynamic type string { + source: content + dynamic + } + } + +} diff --git a/config-model/src/test/derived/ngram/index-info.cfg b/config-model/src/test/derived/ngram/index-info.cfg new file mode 100644 index 00000000000..72b6760ceb5 --- /dev/null +++ b/config-model/src/test/derived/ngram/index-info.cfg @@ -0,0 +1,21 @@ +indexinfo[].name "chunk" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "string" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "ngram 3" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "string" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "ngram 3" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "dynteaser" diff --git a/config-model/src/test/examples/indexing_attribute_changed.sd b/config-model/src/test/examples/indexing_attribute_changed.sd deleted file mode 100644 index bab878d09ab..00000000000 --- a/config-model/src/test/examples/indexing_attribute_changed.sd +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_attribute_changed { - document indexing_attribute_changed { - field foo type string { - indexing: summary | lowercase | attribute - } - } -} diff --git a/config-model/src/test/examples/indexing_attribute_other.sd b/config-model/src/test/examples/indexing_attribute_other.sd deleted file mode 100644 index e3f58f20910..00000000000 --- a/config-model/src/test/examples/indexing_attribute_other.sd +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_attribute_other { - document indexing_attribute_other { - field foo type string { - indexing: attribute bar - } - } -} diff --git a/config-model/src/test/examples/indexing_extra_field_input_extra_field.sd b/config-model/src/test/examples/indexing_extra_field_input_extra_field.sd deleted file mode 100644 index 315d6c2a677..00000000000 --- a/config-model/src/test/examples/indexing_extra_field_input_extra_field.sd +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_extra_field_input_extra_field { - document indexing_extra_field_input_extra_field { - - } - field foo type string { - - } - field bar type string { - indexing: input bar | index - } -} diff --git a/config-model/src/test/examples/indexing_extra_field_input_implicit.sd b/config-model/src/test/examples/indexing_extra_field_input_implicit.sd deleted file mode 100644 index 8aff3284ce3..00000000000 --- a/config-model/src/test/examples/indexing_extra_field_input_implicit.sd +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_extra_field_input_implicit { - document indexing_extra_field_input_implicit { - - } - field foo type string { - indexing: index - } -} diff --git a/config-model/src/test/examples/indexing_extra_field_input_null.sd b/config-model/src/test/examples/indexing_extra_field_input_null.sd deleted file mode 100644 index c4600fa680a..00000000000 --- a/config-model/src/test/examples/indexing_extra_field_input_null.sd +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_extra_field_input_null { - document indexing_extra_field_input_null { - - } - field foo type string { - indexing: input foo | index - } -} diff --git a/config-model/src/test/examples/indexing_extra_field_input_self.sd b/config-model/src/test/examples/indexing_extra_field_input_self.sd deleted file mode 100644 index 36dbae21449..00000000000 --- a/config-model/src/test/examples/indexing_extra_field_input_self.sd +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_extra_field_input_self { - document indexing_extra_field_input_self { - - } - field foo type string { - indexing: input foo | index - } -} diff --git a/config-model/src/test/examples/indexing_index_changed.sd b/config-model/src/test/examples/indexing_index_changed.sd deleted file mode 100644 index 194a9bd3177..00000000000 --- a/config-model/src/test/examples/indexing_index_changed.sd +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_index_changed { - document indexing_index_changed { - field foo type string { - indexing: attribute | lowercase | index - } - } -} diff --git a/config-model/src/test/examples/indexing_index_other.sd b/config-model/src/test/examples/indexing_index_other.sd deleted file mode 100644 index 40b8b150a5b..00000000000 --- a/config-model/src/test/examples/indexing_index_other.sd +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_index_other { - document indexing_index_other { - field foo type string { - indexing: index bar - } - } -} diff --git a/config-model/src/test/examples/indexing_modify_field_no_output.sd b/config-model/src/test/examples/indexing_modify_field_no_output.sd deleted file mode 100644 index ac2bed520e8..00000000000 --- a/config-model/src/test/examples/indexing_modify_field_no_output.sd +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_modify_field_no_output { - document indexing_modify_field_no_output { - field foo type string { - indexing: lowercase | echo - } - } -} diff --git a/config-model/src/test/examples/indexing_output_conflict.sd b/config-model/src/test/examples/indexing_output_conflict.sd deleted file mode 100644 index 9cf1cbc0823..00000000000 --- a/config-model/src/test/examples/indexing_output_conflict.sd +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_output_confict { - document indexing_output_confict { - field foo type string { - - } - } - field bar type string { - indexing: input foo | attribute | lowercase | index - } -} diff --git a/config-model/src/test/examples/indexing_output_other_field.sd b/config-model/src/test/examples/indexing_output_other_field.sd deleted file mode 100644 index 40b08bb15b2..00000000000 --- a/config-model/src/test/examples/indexing_output_other_field.sd +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_output_other_field { - document indexing_output_other_field { - field foo type string { - indexing: index bar - } - } - field bar type string { - - } -} diff --git a/config-model/src/test/examples/indexing_summary_other.sd b/config-model/src/test/examples/indexing_summary_other.sd deleted file mode 100644 index 871ab854c51..00000000000 --- a/config-model/src/test/examples/indexing_summary_other.sd +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_summary_other { - document indexing_summary_other { - field foo type string { - indexing: summary bar - } - } -} diff --git a/config-model/src/test/examples/matchphase/non_existing_attribute.sd b/config-model/src/test/examples/matchphase/non_existing_attribute.sd deleted file mode 100644 index cd3842fde8a..00000000000 --- a/config-model/src/test/examples/matchphase/non_existing_attribute.sd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search test { - document test { - field foo type int { - indexing: summary - } - } - rank-profile default { - match-phase { - attribute: foo - max-hits: 100 - } - } -} diff --git a/config-model/src/test/examples/matchphase/non_fast_search_attribute.sd b/config-model/src/test/examples/matchphase/non_fast_search_attribute.sd deleted file mode 100644 index 5fde096cf61..00000000000 --- a/config-model/src/test/examples/matchphase/non_fast_search_attribute.sd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search test { - document test { - field foo type int { - indexing: attribute - } - } - rank-profile default { - match-phase { - attribute: foo - max-hits: 100 - } - } -} diff --git a/config-model/src/test/examples/matchphase/wrong_collection_type_attribute.sd b/config-model/src/test/examples/matchphase/wrong_collection_type_attribute.sd deleted file mode 100644 index 8a9166c94f7..00000000000 --- a/config-model/src/test/examples/matchphase/wrong_collection_type_attribute.sd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search test { - document test { - field foo type array<int> { - indexing: attribute - } - } - rank-profile default { - match-phase { - attribute: foo - max-hits: 100 - } - } -} diff --git a/config-model/src/test/examples/matchphase/wrong_data_type_attribute.sd b/config-model/src/test/examples/matchphase/wrong_data_type_attribute.sd deleted file mode 100644 index d4f526569ea..00000000000 --- a/config-model/src/test/examples/matchphase/wrong_data_type_attribute.sd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search test { - document test { - field foo type string { - indexing: attribute - } - } - rank-profile default { - match-phase { - attribute: foo - max-hits: 100 - } - } -} diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java index af05a144b79..3f5173a3ae9 100644 --- a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java +++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -84,4 +85,6 @@ public class MockModelContext implements ModelContext { public ExecutorService getExecutor() { return new InThreadExecutorService(); } + + @Override public OnnxModelCost onnxModelCost() { return OnnxModelCost.disabled(); } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 2f8a8bddf20..38f51323ee2 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -148,7 +148,7 @@ public class ModelProvisioningTest { assertEquals("-Xlog:gc", mydisc2.getContainers().get(1).getJvmOptions()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(0).getPreLoad()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(1).getPreLoad()); - assertEquals(Optional.of(45), mydisc2.getMemoryPercentage()); + assertEquals(45, mydisc2.getMemoryPercentage().get().percentage()); assertEquals(Optional.of("-XX:+UseParNewGC"), mydisc2.getJvmGCOptions()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); mydisc2.getConfig(qrStartBuilder); @@ -288,10 +288,11 @@ public class ModelProvisioningTest { assertEquals(2025077080L, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); - assertEquals(1, logger.msgs().size()); + var msgs = logger.msgs().stream().filter(m -> m.level().equals(Level.WARNING)).toList(); + assertEquals(1, msgs.size()); assertEquals("Declaring combined cluster with <nodes of=\"...\"> is deprecated without replacement, " + "and the feature will be removed in Vespa 9. Use separate container and content clusters instead", - logger.msgs().get(0).message); + msgs.get(0).message); } @Test diff --git a/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java index 3ca182e18c2..db862dd388e 100644 --- a/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java @@ -21,7 +21,7 @@ import static org.junit.jupiter.api.Assertions.*; /** * Attribute settings * - * @author bratseth + * @author bratseth */ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { diff --git a/config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java new file mode 100644 index 00000000000..4481445858a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java @@ -0,0 +1,15 @@ +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +public class NGramTestCase extends AbstractExportingTestCase { + + @Test + void testNGram() throws IOException, ParseException { + assertCorrectDeriving("ngram"); + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java index e69f26a31c9..89eff4ec464 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java @@ -269,7 +269,6 @@ public class SchemaParserTestCase { checkFileParses("src/test/examples/implicitsummaryfields.sd"); checkFileParses("src/test/examples/incorrectrankingexpressionfileref.sd"); checkFileParses("src/test/examples/indexing_extra.sd"); - checkFileParses("src/test/examples/indexing_modify_field_no_output.sd"); checkFileParses("src/test/examples/indexing.sd"); checkFileParses("src/test/examples/indexrewrite.sd"); checkFileParses("src/test/examples/indexsettings.sd"); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java b/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java index f9c1e992347..36f20c18588 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java @@ -38,6 +38,6 @@ public abstract class AssertIndexingScript { String str = actualExp.toString(); assertTrue(parsedExpected.remove(str), "Unexpected: " + str); } - assertTrue(parsedExpected.isEmpty(), "Missing: " + parsedExpected.toString()); + assertTrue(parsedExpected.isEmpty(), "Missing: " + parsedExpected); } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java b/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java deleted file mode 100644 index 12da3f0797b..00000000000 --- a/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.processing; - -import com.yahoo.schema.ApplicationBuilder; -import com.yahoo.schema.parser.ParseException; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author Simon Thoresen Hult - */ -public abstract class AssertSearchBuilder { - - public static void assertBuilds(String searchDefinitionFileName) throws IOException, ParseException { - assertNotNull(ApplicationBuilder.buildFromFile(searchDefinitionFileName)); - } - - public static void assertBuildFails(String searchDefinitionFileName, String expectedException) - throws IOException, ParseException { - try { - ApplicationBuilder.buildFromFile(searchDefinitionFileName); - fail(searchDefinitionFileName); - } catch (IllegalArgumentException e) { - assertEquals(expectedException, e.getMessage()); - } - } -} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java index d420623f233..675168ca6c2 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java @@ -1,45 +1,165 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.processing; +import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; -import java.io.IOException; - -import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * @author Simon Thoresen Hult + * @author bratseth */ public class IndexingInputsTestCase { @Test - void requireThatExtraFieldInputExtraFieldThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_extra_field_input_extra_field.sd", - "For schema 'indexing_extra_field_input_extra_field', field 'bar': Indexing script refers " + - "to field 'bar' which is neither a field in document type " + - "'indexing_extra_field_input_extra_field' nor a mutable attribute"); + void requireThatExtraFieldInputExtraFieldThrows() throws ParseException { + try { + var schema = """ + search indexing_extra_field_input_extra_field { + document indexing_extra_field_input_extra_field { + } + field foo type string { + } + field bar type string { + indexing: input bar | index + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_extra_field_input_extra_field', field 'bar': Indexing script refers " + + "to field 'bar' which is neither a field in document type " + + "'indexing_extra_field_input_extra_field' nor a mutable attribute", + Exceptions.toMessageString(e)); + } + } + + @Test + void requireThatExtraFieldInputImplicitThrows() throws ParseException { + try { + var schema = """ + search indexing_extra_field_input_implicit { + document indexing_extra_field_input_implicit { + } + field foo type string { + indexing: index + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_extra_field_input_implicit', field 'foo': " + + "For expression '{ tokenize normalize stem:\"BEST\" | index foo; }': Expected string input, but no input is specified", + Exceptions.toMessageString(e)); + } + } + + @Test + void requireThatExtraFieldInputNullThrows() throws ParseException { + try { + var schema = """ + search indexing_extra_field_input_null { + document indexing_extra_field_input_null { + } + field foo type string { + indexing: input foo | index + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_extra_field_input_null', field 'foo': Indexing script refers to field " + + "'foo' which is neither a field in document type 'indexing_extra_field_input_null' nor a mutable attribute", + Exceptions.toMessageString(e)); + } + } + + @Test + void requireThatExtraFieldInputSelfThrows() throws ParseException { + try { + var schema = """ + search indexing_extra_field_input_self { + document indexing_extra_field_input_self { + } + field foo type string { + indexing: input foo | index + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_extra_field_input_self', field 'foo': Indexing script refers to field " + + "'foo' which is neither a field in document type 'indexing_extra_field_input_self' nor a mutable attribute", + Exceptions.toMessageString(e)); + } } @Test - void requireThatExtraFieldInputImplicitThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_extra_field_input_implicit.sd", - "For schema 'indexing_extra_field_input_implicit', field 'foo': Indexing script refers to " + - "field 'foo' which is neither a field in document type 'indexing_extra_field_input_implicit' nor a mutable attribute"); + void testPlainInputInDerivedField() throws ParseException { + var schema = """ + schema test { + document test { + field field1 type int { + } + } + field derived1 type int { + indexing: input field1 | attribute + } + } + """; + ApplicationBuilder.createFromString(schema); } @Test - void requireThatExtraFieldInputNullThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_extra_field_input_null.sd", - "For schema 'indexing_extra_field_input_null', field 'foo': Indexing script refers to field " + - "'foo' which is neither a field in document type 'indexing_extra_field_input_null' nor a mutable attribute"); + void testWrappedInputInDerivedField() throws ParseException { + var schema = """ + schema test { + document test { + field field1 type int { + } + } + field derived1 type int { + indexing: if (input field1 == 0) { 0 } else { 1 } | attribute + } + } + """; + ApplicationBuilder.createFromString(schema); } @Test - void requireThatExtraFieldInputSelfThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_extra_field_input_self.sd", - "For schema 'indexing_extra_field_input_self', field 'foo': Indexing script refers to field " + - "'foo' which is neither a field in document type 'indexing_extra_field_input_self' nor a mutable attribute"); + void testNoInputInDerivedField() throws ParseException { + try { + var schema = """ + schema test { + document test { + field field1 type int { + } + } + field derived1 type int { + indexing: attribute + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'derived1': For expression '{ attribute derived1; }': " + + "Expected any input, but no input is specified", + Exceptions.toMessageString(e)); + } } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java index e707d203381..7557ca5b725 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.processing; +import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; -import java.io.IOException; - -import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; - +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * @author Simon Thoresen Hult @@ -15,16 +15,51 @@ import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; public class IndexingOutputsTestCase { @Test - void requireThatOutputOtherFieldThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_output_other_field.sd", - "For schema 'indexing_output_other_field', field 'foo': Indexing expression 'index bar' " + - "attempts to write to a field other than 'foo'."); + void requireThatOutputOtherFieldThrows() throws ParseException { + try { + var schema = """ + search indexing_output_other_field { + document indexing_output_other_field { + field foo type string { + indexing: index bar + } + } + field bar type string { + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_output_other_field', field 'foo': Indexing expression 'index bar' " + + "attempts to write to a field other than 'foo'.", + Exceptions.toMessageString(e)); + } } @Test - void requireThatOutputConflictThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_output_conflict.sd", - "For schema 'indexing_output_confict', field 'bar': For expression 'index bar': Attempting " + - "to assign conflicting values to field 'bar'."); + void requireThatOutputConflictThrows() throws ParseException { + try { + var schema = """ + search indexing_output_confict { + document indexing_output_confict { + field foo type string { + } + } + field bar type string { + indexing: input foo | attribute | lowercase | index + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_output_confict', field 'bar': For expression 'index bar': Attempting " + + "to assign conflicting values to field 'bar'.", + Exceptions.toMessageString(e)); + } } + } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java index aa8a2922e8f..4343abbf548 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java @@ -4,13 +4,15 @@ package com.yahoo.schema.processing; import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.derived.AbstractExportingTestCase; import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.Arrays; import static com.yahoo.schema.processing.AssertIndexingScript.assertIndexing; -import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * @author Simon Thoresen Hult @@ -18,45 +20,135 @@ import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; public class IndexingValidationTestCase extends AbstractExportingTestCase { @Test - void testAttributeChanged() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_attribute_changed.sd", - "For schema 'indexing_attribute_changed', field 'foo': For expression 'attribute foo': " + - "Attempting to assign conflicting values to field 'foo'."); + void testAttributeChanged() throws ParseException { + try { + var schema = """ + search indexing_attribute_changed { + document indexing_attribute_changed { + field foo type string { + indexing: summary | lowercase | attribute + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_attribute_changed', field 'foo': For expression 'attribute foo': " + + "Attempting to assign conflicting values to field 'foo'.", + Exceptions.toMessageString(e)); + } } @Test - void testAttributeOther() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_attribute_other.sd", - "For schema 'indexing_attribute_other', field 'foo': Indexing expression 'attribute bar' " + - "attempts to write to a field other than 'foo'."); + void testAttributeOther() throws ParseException { + try { + var schema = """ + search indexing_attribute_other { + document indexing_attribute_other { + field foo type string { + indexing: attribute bar + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_attribute_other', field 'foo': Indexing expression 'attribute bar' " + + "attempts to write to a field other than 'foo'.", + Exceptions.toMessageString(e)); + } } @Test - void testIndexChanged() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_index_changed.sd", - "For schema 'indexing_index_changed', field 'foo': For expression 'index foo': " + - "Attempting to assign conflicting values to field 'foo'."); + void testIndexChanged() throws ParseException { + try { + var schema = """ + search indexing_index_changed { + document indexing_index_changed { + field foo type string { + indexing: attribute | lowercase | index + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_index_changed', field 'foo': For expression 'index foo': " + + "Attempting to assign conflicting values to field 'foo'.", + Exceptions.toMessageString(e)); + } } @Test - void testIndexOther() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_index_other.sd", - "For schema 'indexing_index_other', field 'foo': Indexing expression 'index bar' " + - "attempts to write to a field other than 'foo'."); + void testIndexOther() throws ParseException { + try { + var schema = """ + search indexing_index_other { + document indexing_index_other { + field foo type string { + indexing: index bar\s + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_index_other', field 'foo': Indexing expression 'index bar' " + + "attempts to write to a field other than 'foo'.", + Exceptions.toMessageString(e)); + } } @Test - void testSummaryChanged() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_summary_changed.sd", - "For schema 'indexing_summary_fail', field 'foo': For expression 'summary foo': Attempting " + - "to assign conflicting values to field 'foo'."); + void testSummaryChanged() throws ParseException { + try { + var schema = """ + search indexing_summary_fail { + document indexing_summary_fail { + field foo type string { + indexing: index | lowercase | summary\s + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_summary_fail', field 'foo': For expression 'summary foo': Attempting " + + "to assign conflicting values to field 'foo'.", + Exceptions.toMessageString(e)); + } } @Test - void testSummaryOther() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_summary_other.sd", - "For schema 'indexing_summary_other', field 'foo': Indexing expression 'summary bar' " + - "attempts to write to a field other than 'foo'."); + void testSummaryOther() throws ParseException { + try { + var schema = """ + search indexing_summary_other { + document indexing_summary_other { + field foo type string { + indexing: summary bar + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_summary_other', field 'foo': Indexing expression 'summary bar' " + + "attempts to write to a field other than 'foo'.", + Exceptions.toMessageString(e)); + } } @Test @@ -68,9 +160,35 @@ public class IndexingValidationTestCase extends AbstractExportingTestCase { } @Test - void requireThatMultilineOutputConflictThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_multiline_output_conflict.sd", - "For schema 'indexing_multiline_output_confict', field 'cox': For expression 'index cox': " + - "Attempting to assign conflicting values to field 'cox'."); + void requireThatMultilineOutputConflictThrows() throws ParseException { + try { + var schema = """ + search indexing_multiline_output_confict { + document indexing_multiline_output_confict { + field foo type string { + } + field bar type string { + } + field baz type string { + } + } + field cox type string { + indexing { + input foo | attribute; + input bar | index; + input baz | summary; + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_multiline_output_confict', field 'cox': For expression 'index cox': " + + "Attempting to assign conflicting values to field 'cox'.", + Exceptions.toMessageString(e)); + } } + } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java index 1f723924db6..c46b4fc2c7d 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java @@ -1,13 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.processing; +import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import java.io.IOException; -import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; -import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuilds; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * @author Simon Thoresen Hult @@ -15,16 +17,43 @@ import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuilds; public class IndexingValuesTestCase { @Test - void requireThatModifyFieldNoOutputDoesNotThrow() throws IOException, ParseException { - assertBuilds("src/test/examples/indexing_modify_field_no_output.sd"); + void requireThatModifyFieldNoOutputDoesNotThrow() throws ParseException { + var schema = """ + search indexing_modify_field_no_output { + document indexing_modify_field_no_output { + field foo type string { + indexing: lowercase | echo + } + } + } + """; + ApplicationBuilder.createFromString(schema); } @Test void requireThatInputOtherFieldThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_input_other_field.sd", - "For schema 'indexing_input_other_field', field 'bar': Indexing expression 'input foo' " + - "attempts to modify the value of the document field 'bar'. " + - "Use a field outside the document block instead."); + try { + var schema = """ + search indexing_input_other_field { + document indexing_input_other_field { + field foo type string { + + } + field bar type string { + indexing: input foo | attribute | index | summary + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_input_other_field', field 'bar': Indexing expression 'input foo' " + + "attempts to modify the value of the document field 'bar'. " + + "Use a field outside the document block instead.", + Exceptions.toMessageString(e)); + } } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java index cbddea8ea6a..85ec80d2610 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java @@ -1,9 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.processing; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; -import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; public class MatchPhaseSettingsValidatorTestCase { @@ -13,25 +16,109 @@ public class MatchPhaseSettingsValidatorTestCase { @Test void requireThatAttributeMustExists() throws Exception { - assertBuildFails("src/test/examples/matchphase/non_existing_attribute.sd", - getMessagePrefix() + "does not exists"); + try { + var schema = """ + search test { + document test { + field foo type int { + indexing: summary + } + } + rank-profile default { + match-phase { + attribute: foo + max-hits: 100 + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals(getMessagePrefix() + "does not exists", Exceptions.toMessageString(e)); + } } @Test void requireThatAttributeMustBeNumeric() throws Exception { - assertBuildFails("src/test/examples/matchphase/wrong_data_type_attribute.sd", - getMessagePrefix() + "must be single value numeric, but it is 'string'"); + try { + var schema = """ + search test { + document test { + field foo type string { + indexing: attribute + } + } + rank-profile default { + match-phase { + attribute: foo + max-hits: 100 + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals(getMessagePrefix() + "must be single value numeric, but it is 'string'", + Exceptions.toMessageString(e)); + } } @Test void requireThatAttributeMustBeSingleValue() throws Exception { - assertBuildFails("src/test/examples/matchphase/wrong_collection_type_attribute.sd", - getMessagePrefix() + "must be single value numeric, but it is 'Array<int>'"); + try { + var schema = """ + search test { + document test { + field foo type array<int> { + indexing: attribute + } + } + rank-profile default { + match-phase { + attribute: foo + max-hits: 100 + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals(getMessagePrefix() + "must be single value numeric, but it is 'Array<int>'", + Exceptions.toMessageString(e)); + } } @Test void requireThatAttributeMustHaveFastSearch() throws Exception { - assertBuildFails("src/test/examples/matchphase/non_fast_search_attribute.sd", - getMessagePrefix() + "must be fast-search, but it is not"); + try { + var schema = """ + search test { + document test { + field foo type int { + indexing: attribute + } + } + rank-profile default { + match-phase { + attribute: foo + max-hits: 100 + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals(getMessagePrefix() + "must be fast-search, but it is not", + Exceptions.toMessageString(e)); + } } + } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java index 06ea202b9c3..551542a84ba 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java @@ -27,15 +27,15 @@ public class NGramTestCase extends AbstractSchemaTestCase { SDField gram1 = schema.getConcreteField("gram_1"); assertEquals(MatchType.GRAM, gram1.getMatching().getType()); - assertEquals(1, gram1.getMatching().getGramSize()); + assertEquals(1, gram1.getMatching().getGramSize().getAsInt()); SDField gram2 = schema.getConcreteField("gram_2"); assertEquals(MatchType.GRAM, gram2.getMatching().getType()); - assertEquals(-1, gram2.getMatching().getGramSize()); // Not set explicitly + assertTrue(gram2.getMatching().getGramSize().isEmpty()); SDField gram3 = schema.getConcreteField("gram_3"); assertEquals(MatchType.GRAM, gram3.getMatching().getType()); - assertEquals(3, gram3.getMatching().getGramSize()); + assertEquals(3, gram3.getMatching().getGramSize().getAsInt()); assertEquals("input gram_1 | ngram 1 | index gram_1 | summary gram_1", gram1.getIndexingScript().iterator().next().toString()); assertEquals("input gram_2 | ngram 2 | attribute gram_2 | index gram_2", gram2.getIndexingScript().iterator().next().toString()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java index 49019e47bc2..eae4f12f62c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java @@ -52,11 +52,12 @@ public class MetricsConsumersTest { @Test void consumers_are_set_up_for_hosted() { ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted); - assertEquals(4, config.consumer().size()); + assertEquals(5, config.consumer().size()); assertEquals(MetricsConsumer.vespa.id(), config.consumer(0).name()); assertEquals(MetricsConsumer.autoscaling.id(), config.consumer(1).name()); assertEquals(MetricsConsumer.defaultConsumer.id(), config.consumer(2).name()); assertEquals(MetricsProxyContainerCluster.NEW_DEFAULT_CONSUMER_ID, config.consumer(3).name()); + assertEquals(MetricsConsumer.vespa9.id(), config.consumer(4).name()); } @Test @@ -124,7 +125,7 @@ public class MetricsConsumersTest { ); VespaModel hostedModel = getModel(services, hosted); ConsumersConfig config = consumersConfigFromModel(hostedModel); - assertEquals(4, config.consumer().size()); + assertEquals(5, config.consumer().size()); // All default metrics are retained ConsumersConfig.Consumer vespaConsumer = config.consumer(0); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java new file mode 100644 index 00000000000..9b3b659c252 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -0,0 +1,130 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.model.VespaModel; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.net.URI; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author bjorncs + */ +class JvmHeapSizeValidatorTest { + + @Test + void fails_on_too_low_jvm_percentage() throws IOException, SAXException { + var deployState = createDeployState(8, 7L * 1024 * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + var e = assertThrows(IllegalArgumentException.class, () -> new JvmHeapSizeValidator().validate(model, deployState)); + String expectedMessage = "Allocated percentage of memory of JVM in cluster 'container' is too low (3% < 15%). Estimated cost of ONNX models is 7.00GB"; + assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); + } + + @Test + void fails_on_too_low_heap_size() throws IOException, SAXException { + var deployState = createDeployState(2.2, 1024L * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + var e = assertThrows(IllegalArgumentException.class, () -> new JvmHeapSizeValidator().validate(model, deployState)); + String expectedMessage = "Allocated memory to JVM in cluster 'container' is too low (0.50GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."; + assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); + } + + @Test + void accepts_adequate_heap_size() throws IOException, SAXException { + var deployState = createDeployState(8, 1024L * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + assertDoesNotThrow(() -> new JvmHeapSizeValidator().validate(model, deployState)); + } + + @Test + void accepts_services_with_explicit_jvm_size() throws IOException, SAXException { + String servicesXml = + """ + <?xml version="1.0" encoding="utf-8" ?> + <services version='1.0'> + <container version='1.0'> + <nodes count="2"> + <jvm allocated-memory='5%'/> + <resources vcpu="4" memory="2Gb" disk="125Gb"/> + </nodes> + <component id="hf-embedder" type="hugging-face-embedder"> + <transformer-model url="https://my/url/model.onnx"/> + <tokenizer-model path="app/tokenizer.json"/> + </component> + </container> + </services>"""; + var deployState = createDeployState(servicesXml, 2, 1024L * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + assertDoesNotThrow(() -> new JvmHeapSizeValidator().validate(model, deployState)); + } + + private static DeployState createDeployState(String servicesXml, double nodeGb, long modelCostBytes) { + return new DeployState.Builder() + .applicationPackage( + new MockApplicationPackage.Builder() + .withServices(servicesXml) + .build()) + .modelHostProvisioner(new InMemoryProvisioner(5, new NodeResources(4, nodeGb, 125, 0.3), true)) + .properties(new TestProperties().setHostedVespa(true).setDynamicHeapSize(true)) + .onnxModelCost(new ModelCostDummy(modelCostBytes)) + .build(); + } + + private static DeployState createDeployState(double nodeGb, long modelCostBytes) { + String servicesXml = + """ + <?xml version="1.0" encoding="utf-8" ?> + <services version='1.0'> + <container version='1.0'> + <nodes count="2"> + <resources vcpu="4" memory="%fGb" disk="125Gb"/> + </nodes> + <component id="hf-embedder" type="hugging-face-embedder"> + <transformer-model url="https://my/url/model.onnx"/> + <tokenizer-model path="app/tokenizer.json"/> + </component> + </container> + </services>""".formatted(nodeGb); + return createDeployState(servicesXml, nodeGb, modelCostBytes); + } + + private static class ModelCostDummy implements OnnxModelCost, OnnxModelCost.Calculator { + final AtomicLong totalCost = new AtomicLong(); + final long modelCost; + + ModelCostDummy(long modelCost) { this.modelCost = modelCost; } + + @Override public Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger) { return this; } + @Override public long aggregatedModelCostInBytes() { return totalCost.get(); } + @Override public void registerModel(ApplicationFile path) {} + + @SuppressWarnings("removal") @Override public void registerModel(ModelReference ref) {} + + @Override + public void registerModel(URI uri) { + assertEquals("https://my/url/model.onnx", uri.toString()); + totalCost.addAndGet(modelCost); + } + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java new file mode 100644 index 00000000000..cef4d8c27dd --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java @@ -0,0 +1,107 @@ +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.embedding.BertBaseEmbedderConfig; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.model.VespaModel; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static com.yahoo.config.provision.Environment.prod; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class UrlConfigValidatorTest { + + @Test + void failsWhenContainerNodesNotExclusive() throws IOException, SAXException { + runValidatorOnApp(true, SystemName.Public); // Exclusive nodes in public => success + + assertEquals("Found s3:// urls in config for container cluster default. This is only supported in public systems", + assertThrows(IllegalArgumentException.class, + () -> runValidatorOnApp(false, SystemName.main)) + .getMessage()); + + assertEquals("Found s3:// urls in config for container cluster default. This is only supported in public systems", + assertThrows(IllegalArgumentException.class, + () -> runValidatorOnApp(true, SystemName.main)) + .getMessage()); + + assertEquals("Found s3:// urls in config for container cluster default. Nodes in the cluster need to be 'exclusive'," + + " see https://cloud.vespa.ai/en/reference/services#nodes", + assertThrows(IllegalArgumentException.class, + () -> runValidatorOnApp(false, SystemName.Public)) + .getMessage()); + } + + private static String containerXml(boolean isExclusive) { + return """ + <container version='1.0' id='default'> + <component id='transformer' class='ai.vespa.embedding.BertBaseEmbedder' bundle='model-integration'> + <config name='embedding.bert-base-embedder'> + <transformerModel url='s3://models/minilm-l6-v2/sentence_all_MiniLM_L6_v2.onnx' path='foo'/> + <tokenizerVocab url='s3://models/bert-base-uncased.txt'/> + </config> + </component> + <search/> + <document-api/> + <nodes count='2' exclusive='%s' /> + </container> + """.formatted(Boolean.toString(isExclusive)); + } + + private static void runValidatorOnApp(boolean isExclusive, SystemName systemName) throws IOException, SAXException { + String container = containerXml(isExclusive); + String servicesXml = """ + <services version='1.0'> + %s + </services> + """.formatted(container); + ApplicationPackage app = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .build(); + DeployState deployState = createDeployState(app, systemName); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + new UrlConfigValidator().validate(model, deployState); + } + + private static DeployState createDeployState(ApplicationPackage app, SystemName systemName) { + boolean isHosted = true; + var builder = new DeployState.Builder() + .applicationPackage(app) + .zone(new Zone(systemName, prod, RegionName.from("us-east-3"))) + .properties(new TestProperties().setHostedVespa(isHosted)) + .fileRegistry(new MockFileRegistry()); + + Map<ConfigDefinitionKey, ConfigDefinition> defs = new HashMap<>(); + defs.put(new ConfigDefinitionKey(BertBaseEmbedderConfig.CONFIG_DEF_NAME, BertBaseEmbedderConfig.CONFIG_DEF_NAMESPACE), + new ConfigDefinition(BertBaseEmbedderConfig.CONFIG_DEF_NAME, BertBaseEmbedderConfig.CONFIG_DEF_SCHEMA)); + builder.configDefinitionRepo(new ConfigDefinitionRepo() { + @Override + public Map<ConfigDefinitionKey, com.yahoo.vespa.config.buildergen.ConfigDefinition> getConfigDefinitions() { + return defs; + } + + @Override + public com.yahoo.vespa.config.buildergen.ConfigDefinition get(ConfigDefinitionKey key) { + return defs.get(key); + } + }); + return builder.build(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 894fc55c014..3bdb60a0a8d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -42,14 +42,14 @@ import java.util.Objects; import java.util.OptionalInt; import java.util.Set; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.exclusive; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.shared; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.application; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.global; -import static com.yahoo.config.provision.SystemName.cd; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.zone; import static com.yahoo.config.provision.SystemName.main; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Simon Thoresen Hult @@ -365,62 +365,23 @@ public class ContainerClusterTest { @Test void generatesCorrectRoutingInfo() { - // main system: assertNames(main, ApplicationId.from("t1", "a1", "i1"), - Set.of(), + Set.of(new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.i1.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.i1.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), - Set.of(), - List.of("search-cluster.a1.t1.endpoint.suffix")); - - assertNames(main, - ApplicationId.from("t1", "default", "default"), - Set.of(), - List.of("search-cluster.default.t1.endpoint.suffix")); - - assertNames(main, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), + Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar")), + new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4)), + Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), + new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), + new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); - - // cd system: - assertNames(cd, - ApplicationId.from("t1", "a1", "i1"), - Set.of(), - List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(), - List.of("search-cluster.cd.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "default", "default"), - Set.of(), - List.of("search-cluster.cd.default.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), - List.of("search-cluster.cd.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", global, List.of("a.b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), - new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), - new ContainerEndpoint("not-supported", global, List.of("not.supported"), OptionalInt.empty(), exclusive)), - List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); - } private void assertNames(SystemName systemName, ApplicationId appId, Set<ContainerEndpoint> globalEndpoints, List<String> expectedSharedL4Names) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 94d98f526a0..2dbf49ba61c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -59,7 +59,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; -import java.util.logging.Level; import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.test.LinePatternMatcher.containsLineWithPattern; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index 25c6c0ccb1c..9e8e6ff9c71 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -220,6 +220,26 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { } @Test + void threadpool_configuration_allow_new_syntax() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <search>", + " <threadpool>", + " <threads boost=\"10.2\">0.4</threads>", + " <queue>50</queue>", + " </threadpool>", + " </search>", + nodesXml, + "</container>"); + createModel(root, clusterElem); + ContainerThreadpoolConfig config = root.getConfig( + ContainerThreadpoolConfig.class, "default/component/" + SearchHandler.HANDLER_CLASSNAME + "/threadpool@search-handler"); + assertEquals(-10, config.maxThreads()); + assertEquals(-1, config.minThreads()); + assertEquals(-50, config.queueSize()); + } + + @Test void ExecutionFactory_gets_same_chains_config_as_SearchHandler() { createBasicSearchModel(); Component<?, ?> executionFactory = ((SearchHandler) getComponent("default", SearchHandler.HANDLER_CLASSNAME)) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java index bb5ba840c2c..deea1a820d9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java @@ -7,12 +7,14 @@ import com.yahoo.config.ModelReference; import com.yahoo.config.UrlReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.config.model.test.MockRoot; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.model.SimpleConfigProducer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -30,6 +32,7 @@ import static org.junit.jupiter.api.Assertions.fail; /** * @author Ulf Lilleengen + * @author hmusum */ public class UserConfiguredFilesTest { @@ -68,7 +71,10 @@ public class UserConfiguredFilesTest { } private UserConfiguredFiles userConfiguredFiles() { - return new UserConfiguredFiles(fileRegistry, new BaseDeployLogger()); + return new UserConfiguredFiles(fileRegistry, + new BaseDeployLogger(), + new TestProperties(), + new ApplicationContainerCluster.UserConfiguredUrls()); } @BeforeEach @@ -263,7 +269,8 @@ public class UserConfiguredFilesTest { userConfiguredFiles().register(producer); fail("Should have thrown exception"); } catch (IllegalArgumentException e) { - assertEquals("Unable to register file specified in services.xml for config 'mynamespace.myname': No such file or directory 'foo.txt'", e.getMessage()); + assertEquals("Invalid config in services.xml for 'mynamespace.myname': No such file or directory 'foo.txt'", + e.getMessage()); } } @@ -276,7 +283,7 @@ public class UserConfiguredFilesTest { userConfiguredFiles().register(producer); fail("Should have thrown exception"); } catch (IllegalArgumentException e) { - assertEquals("Unable to register file specified in services.xml for config 'mynamespace.myname': Unable to register file for field 'fileVal': Invalid config value '.'", + assertEquals("Invalid config in services.xml for 'mynamespace.myname': Invalid config value '.' for field 'fileVal", e.getMessage()); } } @@ -291,8 +298,7 @@ public class UserConfiguredFilesTest { userConfiguredFiles().register(producer); fail("Should have thrown exception"); } catch (IllegalArgumentException e) { - assertEquals("Unable to register file specified in services.xml for config 'mynamespace.myname': Directory '" + - relativeTempDir + "' is empty", + assertEquals("Invalid config in services.xml for 'mynamespace.myname': Directory '" + relativeTempDir + "' is empty", e.getMessage()); } } diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index e5cb7e8ef54..8a80c485b9d 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -252,5 +252,11 @@ <aws-parameter-store account="foo" aws-region="us-east-1"/> </store> </secret-store> + <search> + <threadpool> + <threads boost="32.0">8.0</threads> + <queue>40.0</queue> + </threadpool> + </search> </container> </services> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index 49e0b0f478d..dd971ec5108 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -102,6 +102,10 @@ public class ApplicationId implements Comparable<ApplicationId> { return tenant.value() + ":" + application.value() + ":" + instance.value(); } + public String toSerializedFormWithoutInstance() { + return tenant.value() + ":" + application.value(); + } + @Override public String toString() { return toShortString(); } @@ -119,6 +123,11 @@ public class ApplicationId implements Comparable<ApplicationId> { return new ApplicationId(TenantName.defaultName(), ApplicationName.defaultName(), InstanceName.defaultName()); } + /** Returns a serialized form of tenant:application to be used with e.g Flags */ + public static String toSerializedForm(TenantName tenant, ApplicationName application) { + return tenant.value() + ":" + application.value(); + } + // TODO: kill this /** Returns a very special application id, which is not equal to any other id. */ public static ApplicationId global() { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 830e47aa549..51fe16fb232 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -19,7 +19,7 @@ public final class ClusterSpec { private final Type type; private final Id id; - /** The group id of these hosts, or empty if this is represents a request for hosts */ + /** The group id of these hosts, or empty if this represents a request for hosts */ private final Optional<Group> groupId; private final Version vespaVersion; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKeyWithTimestamp.java b/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKeyWithTimestamp.java new file mode 100644 index 00000000000..ecc1cf71113 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKeyWithTimestamp.java @@ -0,0 +1,39 @@ +package com.yahoo.config.provision; + +import com.yahoo.jdisc.Timer; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Random; + +/** + * @author gjoranv + */ +public record WireguardKeyWithTimestamp(WireguardKey key, Instant timestamp) { + + public static final int KEY_ROTATION_BASE = 60; + public static final int KEY_ROTATION_VARIANCE = 10; + public static final int KEY_EXPIRY = KEY_ROTATION_BASE + KEY_ROTATION_VARIANCE + 5; + + public WireguardKeyWithTimestamp { + if (key == null) throw new IllegalArgumentException("Wireguard key cannot be null"); + if (timestamp == null) timestamp = Instant.EPOCH; + } + + public static WireguardKeyWithTimestamp from(String key, long msTimestamp) { + return new WireguardKeyWithTimestamp(WireguardKey.from(key), Instant.ofEpochMilli(msTimestamp)); + } + + public boolean isDueForRotation(Timer timer, ChronoUnit unit, Random random) { + return timer.currentTime().isAfter(keyRotationDueAt(unit, random)); + } + + public boolean hasExpired(Timer timer, ChronoUnit unit) { + return timer.currentTime().isAfter(timestamp.plus(KEY_EXPIRY, unit)); + } + + private Instant keyRotationDueAt(ChronoUnit unit, Random random) { + return timestamp.plus(KEY_ROTATION_BASE + random.nextInt(KEY_ROTATION_VARIANCE), unit); + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java index 5d5757ec79a..2959815dd28 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneEndpoint.java @@ -14,9 +14,16 @@ public class ZoneEndpoint { /** * Endpoint service generation. - * Bump this to provision new services, whenever we change regional endpoint names. - * This will cause new endpoint services to be provisioned, with new domain names. - * TODO: wire multiple service IDs to and through the controller. + * <p> + * This is used to transition to a new set of endpoint services, with new domain names. + * The procedure is: + * <ol> + * <li>Start using new endpoint names (in controller code), for <em>all</em> applications.</li> + * <li>Bump the generation counter here; this causes new services to be provisioned.</li> + * <li>Controller configures the new services with the new endpoint names.</li> + * <li>Let users migrate to the new endpoint names.</li> + * <li>Currently missing: clean up obsolete, unused endpoint services.</li> + * </ol> */ public static final int generation = 0; public static final ZoneEndpoint defaultEndpoint = new ZoneEndpoint(true, false, List.of()); diff --git a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java index 3b24e1c1b8d..4f8d42e895b 100644 --- a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java +++ b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java @@ -111,7 +111,7 @@ public class FlagsHandlerTest { }, { "type": "blacklist", - "dimension": "application", + "dimension": "instance", "values": [ "app1", "app2" ] } ], @@ -127,7 +127,7 @@ public class FlagsHandlerTest { // GET on id2 should now return what was put verifySuccessfulRequest(Method.GET, "/data/" + FLAG2.id(), "", - "{\"id\":\"id2\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"app1\",\"app2\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}"); + "{\"id\":\"id2\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"app1\",\"app2\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}"); // The list of flag data should return id1 and id2 verifySuccessfulRequest(Method.GET, "/data", @@ -153,7 +153,7 @@ public class FlagsHandlerTest { // Get all recursivelly displays all flag data verifySuccessfulRequest(Method.GET, "/data?recursive=true", "", - "{\"flags\":[{\"id\":\"id1\",\"rules\":[{\"value\":false}]},{\"id\":\"id2\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"app1\",\"app2\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}]}"); + "{\"flags\":[{\"id\":\"id1\",\"rules\":[{\"value\":false}]},{\"id\":\"id2\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"app1\",\"app2\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}]}"); // Deleting both flags verifySuccessfulRequest(Method.DELETE, "/data/" + FLAG1.id(), "", ""); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 9533f04107d..e675e00b642 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -41,16 +41,18 @@ import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.applicationmodel.InfrastructureApplication; +import com.yahoo.vespa.config.server.application.ActiveTokenFingerprints.Token; +import com.yahoo.vespa.config.server.application.ActiveTokenFingerprintsClient; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase; import com.yahoo.vespa.config.server.application.ApplicationData; import com.yahoo.vespa.config.server.application.ApplicationReindexing; -import com.yahoo.vespa.config.server.application.ApplicationReindexing.Status; import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.application.ClusterReindexing; import com.yahoo.vespa.config.server.application.ClusterReindexingStatusClient; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStream; import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; +import com.yahoo.vespa.config.server.application.ActiveTokenFingerprints; import com.yahoo.vespa.config.server.application.DefaultClusterReindexingStatusClient; import com.yahoo.vespa.config.server.application.FileDistributionStatus; import com.yahoo.vespa.config.server.application.HttpProxy; @@ -129,7 +131,6 @@ import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA import static com.yahoo.vespa.curator.Curator.CompletionWaiter; import static com.yahoo.yolean.Exceptions.uncheck; import static java.nio.file.Files.readAttributes; -import static java.util.Comparator.naturalOrder; /** * The API for managing applications. @@ -159,6 +160,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final Metric metric; private final SecretStoreValidator secretStoreValidator; private final ClusterReindexingStatusClient clusterReindexingStatusClient; + private final ActiveTokenFingerprints activeTokenFingerprints; private final FlagSource flagSource; @Inject @@ -188,6 +190,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye metric, new SecretStoreValidator(secretStore), new DefaultClusterReindexingStatusClient(), + new ActiveTokenFingerprintsClient(), flagSource); } @@ -205,6 +208,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Metric metric, SecretStoreValidator secretStoreValidator, ClusterReindexingStatusClient clusterReindexingStatusClient, + ActiveTokenFingerprints activeTokenFingerprints, FlagSource flagSource) { this.tenantRepository = Objects.requireNonNull(tenantRepository); this.hostProvisioner = Objects.requireNonNull(hostProvisioner); @@ -219,7 +223,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.testerClient = Objects.requireNonNull(testerClient); this.metric = Objects.requireNonNull(metric); this.secretStoreValidator = Objects.requireNonNull(secretStoreValidator); - this.clusterReindexingStatusClient = clusterReindexingStatusClient; + this.clusterReindexingStatusClient = Objects.requireNonNull(clusterReindexingStatusClient); + this.activeTokenFingerprints = Objects.requireNonNull(activeTokenFingerprints); this.flagSource = flagSource; } @@ -237,6 +242,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private SecretStoreValidator secretStoreValidator = new SecretStoreValidator(new SecretStoreProvider().get()); private FlagSource flagSource = new InMemoryFlagSource(); private ConfigConvergenceChecker configConvergenceChecker = new ConfigConvergenceChecker(); + private Map<String, List<Token>> activeTokens = Map.of(); public Builder withTenantRepository(TenantRepository tenantRepository) { this.tenantRepository = tenantRepository; @@ -298,6 +304,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return this; } + public Builder withActiveTokens(Map<String, List<Token>> tokens) { + this.activeTokens = tokens; + return this; + } + public ApplicationRepository build() { return new ApplicationRepository(tenantRepository, tenantRepository.hostProvisionerProvider().getHostProvisioner(), @@ -313,6 +324,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye metric, secretStoreValidator, ClusterReindexingStatusClient.DUMMY_INSTANCE, + __ -> activeTokens, flagSource); } @@ -612,6 +624,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return uncheck(() -> clusterReindexingStatusClient.getReindexingStatus(getApplication(applicationId))); } + public Map<String, List<Token>> activeTokenFingerprints(ApplicationId applicationId) { + return activeTokenFingerprints.get(getApplication(applicationId)); + } + public Long getApplicationGeneration(ApplicationId applicationId) { return getApplication(applicationId).getApplicationGeneration(); } @@ -1030,21 +1046,21 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private Session validateThatLocalSessionIsNotActive(Tenant tenant, long sessionId) { Session session = getLocalSession(tenant, sessionId); if (Session.Status.ACTIVATE.equals(session.getStatus())) { - throw new IllegalArgumentException("Session is active: " + sessionId); + throw new IllegalArgumentException("Session " + sessionId + " for '" + tenant.getName() + "' is active"); } return session; } private Session getLocalSession(Tenant tenant, long sessionId) { Session session = tenant.getSessionRepository().getLocalSession(sessionId); - if (session == null) throw new NotFoundException("Session " + sessionId + " was not found"); + if (session == null) throw new NotFoundException("Local session " + sessionId + " for '" + tenant.getName() + "' was not found"); return session; } private RemoteSession getRemoteSession(Tenant tenant, long sessionId) { RemoteSession session = tenant.getSessionRepository().getRemoteSession(sessionId); - if (session == null) throw new NotFoundException("Session " + sessionId + " was not found"); + if (session == null) throw new NotFoundException("Remote session " + sessionId + " for '" + tenant.getName() + "' was not found"); return session; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/FallbackOnnxModelCostProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/FallbackOnnxModelCostProvider.java new file mode 100644 index 00000000000..57cfb1cd43b --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/FallbackOnnxModelCostProvider.java @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.config.server; + +import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.container.di.componentgraph.Provider; + +/** + * Default provider that provides a disabled {@link OnnxModelCost} instance. + * + * @author bjorncs + */ +public class FallbackOnnxModelCostProvider implements Provider<OnnxModelCost> { + @Override public OnnxModelCost get() { return OnnxModelCost.disabled(); } + @Override public void deconstruct() {} +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprints.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprints.java new file mode 100644 index 00000000000..9cde5e38302 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprints.java @@ -0,0 +1,18 @@ +package com.yahoo.vespa.config.server.application; + +import com.yahoo.vespa.config.server.modelfactory.ModelResult; + +import java.util.List; +import java.util.Map; + +/** + * @author jonmv + */ +public interface ActiveTokenFingerprints { + + /** Lists all active tokens and their fingerprints for each token-enabled container host in the application, that is currently up. */ + Map<String, List<Token>> get(ModelResult application); + + record Token(String id, List<String> fingerprints) { } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClient.java new file mode 100644 index 00000000000..4e9eac7a9a6 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClient.java @@ -0,0 +1,123 @@ +package com.yahoo.vespa.config.server.application; + +import ai.vespa.http.DomainName; +import ai.vespa.http.HttpURL; +import ai.vespa.http.HttpURL.Path; +import ai.vespa.http.HttpURL.Scheme; +import ai.vespa.util.http.hc5.VespaAsyncHttpClientBuilder; +import com.yahoo.config.model.api.ApplicationClusterEndpoint.AuthMethod; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.slime.Inspector; +import com.yahoo.vespa.config.server.modelfactory.ModelResult; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.core5.concurrent.FutureCallback; +import org.apache.hc.core5.io.CloseMode; +import org.apache.hc.core5.reactor.IOReactorConfig; +import org.apache.hc.core5.util.TimeValue; +import org.apache.hc.core5.util.Timeout; + +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Phaser; + +import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; +import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; +import static com.yahoo.slime.SlimeUtils.entriesStream; +import static com.yahoo.slime.SlimeUtils.jsonToSlime; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; + +/** + * @author jonmv + */ +public class ActiveTokenFingerprintsClient implements ActiveTokenFingerprints, AutoCloseable { + + private final CloseableHttpAsyncClient httpClient = createHttpClient(); + + public ActiveTokenFingerprintsClient() { + httpClient.start(); + } + + @Override + public Map<String, List<Token>> get(ModelResult application) { + Set<String> containersWithTokenFilter = application.getModel().applicationClusterInfo().stream() + .flatMap(cluster -> cluster.endpoints().stream()) + .filter(endpoint -> endpoint.authMethod() == AuthMethod.token) + .flatMap(endpoint -> endpoint.hostNames().stream()) + .collect(toSet()); + return getFingerprints(application.getModel().getHosts().stream() + .filter(host -> containersWithTokenFilter.contains(host.getHostname())) + .flatMap(host -> host.getServices().stream()) + .filter(service -> service.getServiceType().equals(CONTAINER.serviceName) + || service.getServiceType().equals(QRSERVER.serviceName)) + .toList()); + } + + private Map<String, List<Token>> getFingerprints(List<ServiceInfo> services) { + Map<String, List<Token>> tokens = new ConcurrentHashMap<>(); + Phaser phaser = new Phaser(services.size() + 1); + for (ServiceInfo service : services) getFingerprints(tokens, service, phaser); + phaser.arriveAndAwaitAdvance(); + return tokens; + } + + // A container may be unable to provide its fingerprints for a number of reasons, which may be OK, so + // we only track those containers which return an OK response, but we do require at least one such response. + private void getFingerprints(Map<String, List<Token>> hostTokens, ServiceInfo service, Phaser phaser) { + URI uri = HttpURL.create(Scheme.http, + DomainName.of(service.getHostName()), + service.getPorts().stream().filter(port -> port.getTags().stream().anyMatch("http"::equals)).findAny().get().getPort(), + Path.parse("/data-plane-tokens/v1")) + .asURI(); + httpClient.execute(SimpleRequestBuilder.get(uri).build(), new FutureCallback<>() { + @Override public void completed(SimpleHttpResponse result) { + if (result.getCode() == 200) hostTokens.put(service.getHostName(), parseTokens(result)); + phaser.arrive(); + } + @Override public void failed(Exception ex) { phaser.arrive(); } + @Override public void cancelled() { phaser.arrive(); } + }); + } + + private static List<Token> parseTokens(SimpleHttpResponse response) { + return entriesStream(jsonToSlime(response.getBodyBytes()).get().field("tokens")) + .map(entry -> new Token(entry.field("id").asString(), + entriesStream(entry.field("fingerprints")).map(Inspector::asString).toList())) + .toList(); + } + + private static CloseableHttpAsyncClient createHttpClient() { + return VespaAsyncHttpClientBuilder + .create(tlsStrategy -> PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .setDefaultConnectionConfig(ConnectionConfig.custom() + .setConnectTimeout(Timeout.ofSeconds(2)) + .build()) + .build()) + .setIOReactorConfig(IOReactorConfig.custom() + .setSoTimeout(Timeout.ofSeconds(2)) + .build()) + .setDefaultRequestConfig( + RequestConfig.custom() + .setConnectionRequestTimeout(Timeout.ofSeconds(2)) + .setResponseTimeout(Timeout.ofSeconds(2)) + .build()) + .setUserAgent("data-plane-token-client") + .build(); + } + + @Override + public void close() throws Exception { + httpClient.close(CloseMode.GRACEFUL); + httpClient.awaitShutdown(TimeValue.ofSeconds(10)); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 693252da43a..ff2c137c11c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -23,7 +23,6 @@ import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.curator.CompletionTimeoutException; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.curator.transaction.CuratorTransaction; @@ -37,7 +36,6 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.time.Clock; import java.time.Duration; -import java.time.Instant; import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; @@ -62,6 +60,8 @@ import static java.util.stream.Collectors.toSet; public class TenantApplications implements RequestHandler, HostValidator { private static final Logger log = Logger.getLogger(TenantApplications.class.getName()); + /* Time to wait for all config servers to get event when an application is removed */ + private static final Duration waitForAll = Duration.ofSeconds(5); private final Curator curator; private final ApplicationCuratorDatabase database; @@ -430,146 +430,17 @@ public class TenantApplications implements RequestHandler, HostValidator { public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; } public CompletionWaiter createRemoveApplicationWaiter(ApplicationId applicationId) { - return RemoveApplicationWaiter.createAndInitialize(curator, applicationId, serverId); + return curator.createCompletionWaiter(barrierPath(applicationId), serverId, waitForAll); } public CompletionWaiter getRemoveApplicationWaiter(ApplicationId applicationId) { - return RemoveApplicationWaiter.create(curator, applicationId, serverId); + return curator.getCompletionWaiter(barrierPath(applicationId), serverId, waitForAll); } - /** - * Waiter for removing application. Will wait for some time for all servers to remove application, - * but will accept the majority of servers to have removed app if it takes a long time. - */ - // TODO: Merge with CuratorCompletionWaiter - static class RemoveApplicationWaiter implements CompletionWaiter { - - private static final java.util.logging.Logger log = Logger.getLogger(RemoveApplicationWaiter.class.getName()); - private static final Duration waitForAllDefault = Duration.ofSeconds(5); - - private final Curator curator; - private final Path barrierPath; - private final Path waiterNode; - private final Duration waitForAll; - private final Clock clock = Clock.systemUTC(); - - RemoveApplicationWaiter(Curator curator, ApplicationId applicationId, String serverId) { - this(curator, applicationId, serverId, waitForAllDefault); - } - - RemoveApplicationWaiter(Curator curator, ApplicationId applicationId, String serverId, Duration waitForAll) { - this.barrierPath = TenantRepository.getBarriersPath().append(applicationId.tenant().value()) - .append("delete-application") - .append(applicationId.serializedForm()); - this.waiterNode = barrierPath.append(serverId); - this.curator = curator; - this.waitForAll = waitForAll; - } - - @Override - public void awaitCompletion(Duration timeout) { - List<String> respondents; - try { - respondents = awaitInternal(timeout); - } catch (Exception e) { - throw new RuntimeException(e); - } - if (respondents.size() < barrierMemberCount()) { - throw new CompletionTimeoutException("Timed out waiting for peer config servers to remove application " + - "(waited for barrier " + barrierPath + ")." + - "Got response from " + respondents + ", but need response from " + - "at least " + barrierMemberCount() + " server(s). " + - "Timeout passed as argument was " + timeout.toMillis() + " ms"); - } - } - - private List<String> awaitInternal(Duration timeout) throws Exception { - Instant startTime = clock.instant(); - Instant endTime = startTime.plus(timeout); - Instant gotQuorumTime = Instant.EPOCH; - List<String> respondents; - do { - respondents = curator.framework().getChildren().forPath(barrierPath.getAbsolute()); - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, respondents.size() + "/" + curator.zooKeeperEnsembleCount() + " responded: " + - respondents + ", all participants: " + curator.zooKeeperEnsembleConnectionSpec()); - } - - // If all config servers responded, return - if (respondents.size() == curator.zooKeeperEnsembleCount()) { - logBarrierCompleted(respondents, startTime); - break; - } - - // If some are missing, quorum is enough, but wait for all up to 5 seconds before returning - if (respondents.size() >= barrierMemberCount()) { - if (gotQuorumTime.isBefore(startTime)) - gotQuorumTime = clock.instant(); - - // Give up if more than some time has passed since we got quorum, otherwise continue - if (Duration.between(clock.instant(), gotQuorumTime.plus(waitForAll)).isNegative()) { - logBarrierCompleted(respondents, startTime); - break; - } - } - - Thread.sleep(100); - } while (clock.instant().isBefore(endTime)); - - return respondents; - } - - private void logBarrierCompleted(List<String> respondents, Instant startTime) { - Duration duration = Duration.between(startTime, Instant.now()); - Level level = (duration.minus(Duration.ofSeconds(5))).isNegative() ? Level.FINE : Level.INFO; - log.log(level, () -> barrierCompletedMessage(respondents, duration)); - } - - private String barrierCompletedMessage(List<String> respondents, Duration duration) { - return barrierPath + " completed in " + duration.toString() + - ", " + respondents.size() + "/" + curator.zooKeeperEnsembleCount() + " responded: " + respondents; - } - - @Override - public void notifyCompletion() { - try { - curator.framework().create().forPath(waiterNode.getAbsolute()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public String toString() { return "'" + barrierPath + "', " + barrierMemberCount() + " members"; } - - public static CompletionWaiter create(Curator curator, ApplicationId applicationId, String serverId) { - return new RemoveApplicationWaiter(curator, applicationId, serverId); - } - - public static CompletionWaiter create(Curator curator, ApplicationId applicationId, String serverId, Duration waitForAll) { - return new RemoveApplicationWaiter(curator, applicationId, serverId, waitForAll); - } - - public static CompletionWaiter createAndInitialize(Curator curator, ApplicationId applicationId, String serverId) { - return createAndInitialize(curator, applicationId, serverId, waitForAllDefault); - } - - public static CompletionWaiter createAndInitialize(Curator curator, ApplicationId applicationId, String serverId, Duration waitForAll) { - RemoveApplicationWaiter waiter = new RemoveApplicationWaiter(curator, applicationId, serverId, waitForAll); - - // Cleanup and create a new barrier path - Path barrierPath = waiter.barrierPath(); - curator.delete(barrierPath); - curator.create(barrierPath.getParentPath()); - curator.createAtomically(barrierPath); - - return waiter; - } - - private int barrierMemberCount() { return (curator.zooKeeperEnsembleCount() / 2) + 1; /* majority */ } - - private Path barrierPath() { return barrierPath; } - + private static Path barrierPath(ApplicationId applicationId) { + return TenantRepository.getBarriersPath().append(applicationId.tenant().value()) + .append("delete-application") + .append(applicationId.serializedForm()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index 154d2d0f2f0..042aa2423f3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -11,6 +11,7 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -35,15 +36,17 @@ public class DeployHandlerLogger implements DeployLogger { this.logroot = slime.setObject().setArray("log"); } + @Override public void log(Level level, String message) { log(level, () -> message); } + @Override public void log(Level level, Supplier<String> message) { log(level, message, null); } + @Override @SuppressWarnings("deprecation") - public void log(Level level, String message) { - if (level.intValue() <= LogLevel.DEBUG.intValue() && !verbose) - return; + public void log(Level level, Supplier<String> supplier, Throwable throwable) { + // Also tee to a normal log, Vespa log for example, but use level fine + log.log(Level.FINE, throwable, () -> prefix + supplier.get()); - logJson(level, message); - // Also tee to a normal log, Vespa log for example, but use level fine - log.log(Level.FINE, () -> prefix + message); + if (level.intValue() <= LogLevel.DEBUG.intValue() && !verbose) return; + logJson(level, supplier.get()); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 142f98e13e3..96b0b03c832 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -13,6 +13,7 @@ import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.api.Quota; import com.yahoo.config.model.api.Reindexing; @@ -28,6 +29,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.tenant.SecretStoreExternalIdRetriever; import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flag; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; @@ -66,6 +68,7 @@ public class ModelContextImpl implements ModelContext { private final Optional<? extends Reindexing> reindexing; private final ModelContext.Properties properties; private final Optional<File> appDir; + private final OnnxModelCost onnxModelCost; private final Optional<DockerImage> wantedDockerImageRepository; @@ -92,6 +95,7 @@ public class ModelContextImpl implements ModelContext { Provisioned provisioned, ModelContext.Properties properties, Optional<File> appDir, + OnnxModelCost onnxModelCost, Optional<DockerImage> wantedDockerImageRepository, Version modelVespaVersion, Version wantedNodeVespaVersion) { @@ -109,6 +113,7 @@ public class ModelContextImpl implements ModelContext { this.wantedDockerImageRepository = wantedDockerImageRepository; this.modelVespaVersion = modelVespaVersion; this.wantedNodeVespaVersion = wantedNodeVespaVersion; + this.onnxModelCost = onnxModelCost; } @Override @@ -150,6 +155,8 @@ public class ModelContextImpl implements ModelContext { @Override public Optional<File> appDir() { return appDir; } + @Override public OnnxModelCost onnxModelCost() { return onnxModelCost; } + @Override public Optional<DockerImage> wantedDockerImageRepo() { return wantedDockerImageRepository; } @@ -201,6 +208,8 @@ public class ModelContextImpl implements ModelContext { private final boolean enableNestedMultivalueGrouping; private final boolean useReconfigurableDispatcher; private final int contentLayerMetadataFeatureLevel; + private final boolean dynamicHeapSize; + private final String unknownConfigDefinition; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -243,6 +252,8 @@ public class ModelContextImpl implements ModelContext { this.enableNestedMultivalueGrouping = flagValue(source, appId, version, Flags.ENABLE_NESTED_MULTIVALUE_GROUPING); this.useReconfigurableDispatcher = flagValue(source, appId, version, Flags.USE_RECONFIGURABLE_DISPATCHER); this.contentLayerMetadataFeatureLevel = flagValue(source, appId, version, Flags.CONTENT_LAYER_METADATA_FEATURE_LEVEL); + this.dynamicHeapSize = flagValue(source, appId, version, Flags.DYNAMIC_HEAP_SIZE); + this.unknownConfigDefinition = flagValue(source, appId, version, Flags.UNKNOWN_CONFIG_DEFINITION); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -293,10 +304,13 @@ public class ModelContextImpl implements ModelContext { @Override public boolean enableNestedMultivalueGrouping() { return enableNestedMultivalueGrouping; } @Override public boolean useReconfigurableDispatcher() { return useReconfigurableDispatcher; } @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } + @Override public boolean dynamicHeapSize() { return dynamicHeapSize; } + @Override public String unknownConfigDefinition() { return unknownConfigDefinition; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm()) + .with(FetchVector.Dimension.APPLICATION_ID, appId.toSerializedFormWithoutInstance()) .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) .with(FetchVector.Dimension.TENANT_ID, appId.tenant().value()) .boxedValue(); @@ -309,6 +323,7 @@ public class ModelContextImpl implements ModelContext { UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm()) + .with(FetchVector.Dimension.APPLICATION_ID, appId.toSerializedFormWithoutInstance()) .with(FetchVector.Dimension.CLUSTER_TYPE, clusterType.name()) .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) .boxedValue(); @@ -321,6 +336,7 @@ public class ModelContextImpl implements ModelContext { UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm()) + .with(FetchVector.Dimension.APPLICATION_ID, appId.toSerializedFormWithoutInstance()) .with(FetchVector.Dimension.CLUSTER_ID, clusterId.value()) .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) .boxedValue(); @@ -397,21 +413,16 @@ public class ModelContextImpl implements ModelContext { this.tenantSecretStores = tenantSecretStores; this.secretStore = secretStore; this.jvmGCOptionsFlag = PermanentFlags.JVM_GC_OPTIONS.bindTo(flagSource) - .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()); - this.allowDisableMtls = PermanentFlags.ALLOW_DISABLE_MTLS.bindTo(flagSource) - .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value(); + .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.toSerializedFormWithoutInstance()); + this.allowDisableMtls = flagValue(flagSource, applicationId, PermanentFlags.ALLOW_DISABLE_MTLS); this.operatorCertificates = operatorCertificates; - this.tlsCiphersOverride = PermanentFlags.TLS_CIPHERS_OVERRIDE.bindTo(flagSource) - .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value(); + this.tlsCiphersOverride = flagValue(flagSource, applicationId, PermanentFlags.TLS_CIPHERS_OVERRIDE); this.zoneDnsSuffixes = configserverConfig.zoneDnsSuffixes(); - this.environmentVariables = PermanentFlags.ENVIRONMENT_VARIABLES.bindTo(flagSource) - .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value(); + this.environmentVariables = flagValue(flagSource, applicationId, PermanentFlags.ENVIRONMENT_VARIABLES); this.cloudAccount = cloudAccount; - this.allowUserFilters = PermanentFlags.ALLOW_USER_FILTERS.bindTo(flagSource) - .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value(); - this.endpointConnectionTtl = Duration.ofSeconds( - PermanentFlags.ENDPOINT_CONNECTION_TTL.bindTo(flagSource) - .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value()); + this.allowUserFilters = flagValue(flagSource, applicationId, PermanentFlags.ALLOW_USER_FILTERS); + this.endpointConnectionTtl = Duration.ofSeconds(flagValue(flagSource, applicationId, PermanentFlags.ENDPOINT_CONNECTION_TTL)); this.dataplaneTokens = dataplaneTokens; } @@ -512,4 +523,10 @@ public class ModelContextImpl implements ModelContext { @Override public Duration endpointConnectionTtl() { return endpointConnectionTtl; } } + private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { + return flag.bindTo(source) + .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm()) + .with(FetchVector.Dimension.APPLICATION_ID, appId.toSerializedFormWithoutInstance()) + .boxedValue(); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index bd6e0f90b54..f39feceeeb1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -95,9 +95,11 @@ public class ApplicationHandler extends HttpHandler { if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}")) return getApplicationResponse(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/content/{*}")) return content(applicationId(path), path.getRest(), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/filedistributionstatus")) return filedistributionStatus(applicationId(path), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/active-token-fingerprints")) return activeTokenFingerprints(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/logs")) return logs(applicationId(path), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/deployment")) return deploymentMetrics(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/searchnode")) return searchNodeMetrics(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/quota")) return quotaUsage(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return getReindexingStatus(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/status/{*}")) return serviceStatusPage(applicationId(path), path.get("service"), path.get("hostname"), path.getRest(), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/state/v1/{*}")) return serviceStateV1(applicationId(path), path.get("service"), path.get("hostname"), path.getRest(), request); @@ -105,7 +107,6 @@ public class ApplicationHandler extends HttpHandler { if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/serviceconverge/{hostAndPort}")) return checkServiceConverge(applicationId(path), path.get("hostAndPort"), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/suspended")) return isSuspended(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/tester/{command}")) return testerRequest(applicationId(path), path.get("command"), request); - if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/quota")) return quotaUsage(applicationId(path)); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -150,18 +151,11 @@ public class ApplicationHandler extends HttpHandler { } private HttpResponse serviceStatusPage(ApplicationId applicationId, String service, String hostname, HttpURL.Path pathSuffix, HttpRequest request) { - HttpURL.Path pathPrefix = HttpURL.Path.empty(); - switch (service) { - case "container-clustercontroller": - pathPrefix = pathPrefix.append("clustercontroller-status").append("v1"); - break; - case "distributor": - case "storagenode": - pathPrefix = pathPrefix.append("contentnode-status").append("v1"); - break; - default: - throw new com.yahoo.vespa.config.server.NotFoundException("No status page for service: " + service); - } + HttpURL.Path pathPrefix = switch (service) { + case "container-clustercontroller" -> HttpURL.Path.empty().append("clustercontroller-status").append("v1"); + case "distributor", "storagenode" -> HttpURL.Path.empty().append("contentnode-status").append("v1"); + default -> throw new NotFoundException("No status page for service: " + service); + }; return applicationRepository.proxyServiceHostnameRequest(applicationId, hostname, service, pathPrefix.append(pathSuffix), Query.empty().add(request.getJDiscRequest().parameters()), null); } @@ -194,6 +188,22 @@ public class ApplicationHandler extends HttpHandler { return applicationRepository.fileDistributionStatus(applicationId, getTimeoutFromRequest(request)); } + private HttpResponse activeTokenFingerprints(ApplicationId applicationId) { + Slime slime = new Slime(); + Cursor hostsArray = slime.setObject().setArray("hosts"); + applicationRepository.activeTokenFingerprints(applicationId).forEach((host, tokens) -> { + Cursor hostObject = hostsArray.addObject(); + hostObject.setString("host", host); + Cursor tokensArray = hostObject.setArray("tokens"); + tokens.forEach(token -> { + Cursor tokenObject = tokensArray.addObject(); + tokenObject.setString("id", token.id()); + token.fingerprints().forEach(tokenObject.setArray("fingerprints")::addString); + }); + }); + return new SlimeJsonResponse(slime); + } + private HttpResponse logs(ApplicationId applicationId, HttpRequest request) { Optional<DomainName> hostname = Optional.ofNullable(request.getProperty("hostname")).map(DomainName::of); String apiParams = Optional.ofNullable(request.getUri().getQuery()).map(q -> "?" + q).orElse(""); @@ -213,19 +223,13 @@ public class ApplicationHandler extends HttpHandler { } private HttpResponse testerRequest(ApplicationId applicationId, String command, HttpRequest request) { - switch (command) { - case "status": - return applicationRepository.getTesterStatus(applicationId); - case "log": - Long after = Long.valueOf(request.getProperty("after")); - return applicationRepository.getTesterLog(applicationId, after); - case "ready": - return applicationRepository.isTesterReady(applicationId); - case "report": - return applicationRepository.getTestReport(applicationId); - default: - throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString()); - } + return switch (command) { + case "status" -> applicationRepository.getTesterStatus(applicationId); + case "log" -> applicationRepository.getTesterLog(applicationId, Long.valueOf(request.getProperty("after"))); + case "ready" -> applicationRepository.isTesterReady(applicationId); + case "report" -> applicationRepository.getTestReport(applicationId); + default -> throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString()); + }; } private HttpResponse quotaUsage(ApplicationId applicationId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 328bd143d81..d302e0e8008 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -9,6 +9,7 @@ import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.ApplicationId; @@ -58,6 +59,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { private final FlagSource flagSource; private final SecretStore secretStore; private final ExecutorService executor; + private final OnnxModelCost onnxModelCost; public ActivatedModelsBuilder(TenantName tenant, long applicationGeneration, @@ -72,7 +74,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { ConfigserverConfig configserverConfig, Zone zone, ModelFactoryRegistry modelFactoryRegistry, - ConfigDefinitionRepo configDefinitionRepo) { + ConfigDefinitionRepo configDefinitionRepo, + OnnxModelCost onnxModelCost) { super(modelFactoryRegistry, configserverConfig, zone, hostProvisionerProvider, new SilentDeployLogger()); this.tenant = tenant; this.applicationGeneration = applicationGeneration; @@ -84,6 +87,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { this.flagSource = flagSource; this.secretStore = secretStore; this.executor = executor; + this.onnxModelCost = onnxModelCost; } @Override @@ -108,6 +112,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { provisioned, modelContextProperties, Optional.empty(), + onnxModelCost, wantedDockerImageRepository, modelFactory.version(), wantedNodeVespaVersion); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 4faa475fa08..57c766bb9c2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -207,11 +207,12 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { builtModelVersions.add(modelVersion); } catch (RuntimeException e) { // allow failure to create old config models if there is a validation override that allow skipping old - // config models or we're manually deploying + // config models, or we're manually deploying if (builtModelVersions.size() > 0 && ( builtModelVersions.get(0).getModel().skipOldConfigModels(now) || zone().environment().isManuallyDeployed())) - log.log(Level.INFO, applicationId + ": Failed to build version " + version + - ", but allow failure due to validation override or manual deployment"); + log.log(Level.WARNING, applicationId + ": Failed to build version " + version + + ", but allow failure due to validation override or manual deployment:" + + Exceptions.toMessageString(e)); else { log.log(Level.SEVERE, applicationId + ": Failed to build version " + version); throw e; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index af611b131f6..a3f0284890c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -16,6 +16,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.api.ValidationParameters.IgnoreValidationErrors; @@ -69,6 +70,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P private final Optional<ApplicationVersions> activeApplicationVersions; private final Curator curator; private final ExecutorService executor; + private final OnnxModelCost onnxModelCost; public PreparedModelsBuilder(ModelFactoryRegistry modelFactoryRegistry, FlagSource flagSource, @@ -85,7 +87,8 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P PrepareParams params, Optional<ApplicationVersions> activeApplicationVersions, ConfigserverConfig configserverConfig, - Zone zone) { + Zone zone, + OnnxModelCost onnxModelCost) { super(modelFactoryRegistry, configserverConfig, zone, hostProvisionerProvider, deployLogger); this.flagSource = flagSource; this.secretStore = secretStore; @@ -98,6 +101,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P this.params = params; this.activeApplicationVersions = activeApplicationVersions; this.executor = executor; + this.onnxModelCost = onnxModelCost; } @Override @@ -123,6 +127,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P provisioned, createModelContextProperties(modelFactory.version(), applicationPackage), getAppDir(applicationPackage), + onnxModelCost, wantedDockerImageRepository, modelVersion, wantedNodeVespaVersion); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index aeff97169f4..67872865106 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -18,6 +18,7 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.api.Quota; import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.AllocatedHosts; @@ -93,6 +94,7 @@ public class SessionPreparer { private final FlagSource flagSource; private final ExecutorService executor; private final BooleanFlag writeSessionData; + private final OnnxModelCost onnxModelCost; public SessionPreparer(ModelFactoryRegistry modelFactoryRegistry, FileDistributionFactory fileDistributionFactory, @@ -103,7 +105,8 @@ public class SessionPreparer { Curator curator, Zone zone, FlagSource flagSource, - SecretStore secretStore) { + SecretStore secretStore, + OnnxModelCost onnxModelCost) { this.modelFactoryRegistry = modelFactoryRegistry; this.fileDistributionFactory = fileDistributionFactory; this.hostProvisionerProvider = hostProvisionerProvider; @@ -115,6 +118,7 @@ public class SessionPreparer { this.flagSource = flagSource; this.executor = executor; this.writeSessionData = Flags.WRITE_CONFIG_SERVER_SESSION_DATA_AS_ONE_BLOB.bindTo(flagSource); + this.onnxModelCost = onnxModelCost; } ExecutorService getExecutor() { return executor; } @@ -134,7 +138,8 @@ public class SessionPreparer { ApplicationId applicationId = params.getApplicationId(); Preparation preparation = new Preparation(hostValidator, logger, params, activeApplicationVersions, TenantRepository.getTenantPath(applicationId.tenant()), - serverDbSessionDir, applicationPackage, sessionZooKeeperClient); + serverDbSessionDir, applicationPackage, sessionZooKeeperClient, + onnxModelCost); preparation.preprocess(); try { AllocatedHosts allocatedHosts = preparation.buildModels(now); @@ -186,7 +191,7 @@ public class SessionPreparer { Preparation(HostValidator hostValidator, DeployLogger logger, PrepareParams params, Optional<ApplicationVersions> activeApplicationVersions, Path tenantPath, File serverDbSessionDir, ApplicationPackage applicationPackage, - SessionZooKeeperClient sessionZooKeeperClient) { + SessionZooKeeperClient sessionZooKeeperClient, OnnxModelCost onnxModelCost) { this.logger = logger; this.params = params; this.applicationPackage = applicationPackage; @@ -219,7 +224,8 @@ public class SessionPreparer { params, activeApplicationVersions, configserverConfig, - zone); + zone, + onnxModelCost); } void checkTimeout(String step) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 3b57945b21d..eb07e3010c6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -9,6 +9,7 @@ import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.application.provider.DeployData; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; @@ -118,6 +119,7 @@ public class SessionRepository { private final SessionPreparer sessionPreparer; private final Path sessionsPath; private final TenantName tenantName; + private final OnnxModelCost onnxModelCost; private final SessionCounter sessionCounter; private final SecretStore secretStore; private final HostProvisionerProvider hostProvisionerProvider; @@ -147,8 +149,10 @@ public class SessionRepository { Clock clock, ModelFactoryRegistry modelFactoryRegistry, ConfigDefinitionRepo configDefinitionRepo, - int maxNodeSize) { + int maxNodeSize, + OnnxModelCost onnxModelCost) { this.tenantName = tenantName; + this.onnxModelCost = onnxModelCost; sessionCounter = new SessionCounter(curator, tenantName); this.sessionsPath = TenantRepository.getSessionsPath(tenantName); this.clock = clock; @@ -553,7 +557,8 @@ public class SessionRepository { configserverConfig, zone, modelFactoryRegistry, - configDefinitionRepo); + configDefinitionRepo, + onnxModelCost); return ApplicationVersions.fromList(builder.buildModels(session.getApplicationId(), session.getDockerImageRepository(), session.getVespaVersion(), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 2bc8cb5bc0a..378cd9bdb8c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -118,21 +118,21 @@ public class SessionZooKeeperClient { public long sessionId() { return sessionId; } - public CompletionWaiter createActiveWaiter() { return createCompletionWaiter(getWaiterPath(ACTIVE_BARRIER)); } + public CompletionWaiter createActiveWaiter() { return createCompletionWaiter(barrierPath(ACTIVE_BARRIER)); } - CompletionWaiter createPrepareWaiter() { return createCompletionWaiter(getWaiterPath(PREPARE_BARRIER)); } + CompletionWaiter createPrepareWaiter() { return createCompletionWaiter(barrierPath(PREPARE_BARRIER)); } - CompletionWaiter getPrepareWaiter() { return getCompletionWaiter(getWaiterPath(PREPARE_BARRIER)); } + CompletionWaiter getPrepareWaiter() { return getCompletionWaiter(barrierPath(PREPARE_BARRIER)); } - CompletionWaiter getActiveWaiter() { return getCompletionWaiter(getWaiterPath(ACTIVE_BARRIER)); } + CompletionWaiter getActiveWaiter() { return getCompletionWaiter(barrierPath(ACTIVE_BARRIER)); } - CompletionWaiter getUploadWaiter() { return getCompletionWaiter(getWaiterPath(UPLOAD_BARRIER)); } + CompletionWaiter getUploadWaiter() { return getCompletionWaiter(barrierPath(UPLOAD_BARRIER)); } private static final String PREPARE_BARRIER = "prepareBarrier"; private static final String ACTIVE_BARRIER = "activeBarrier"; private static final String UPLOAD_BARRIER = "uploadBarrier"; - private Path getWaiterPath(String barrierName) { + private Path barrierPath(String barrierName) { return sessionPath.append(barrierName); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index ba09b3de365..ea53c8aa2bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -11,6 +11,7 @@ import com.yahoo.concurrent.Locks; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -119,6 +120,7 @@ public class TenantRepository { new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("check for removed applications")); private final Curator.DirectoryCache directoryCache; private final ZookeeperServerConfig zookeeperServerConfig; + private final OnnxModelCost onnxModelCost; /** * Creates a new tenant repository @@ -138,7 +140,8 @@ public class TenantRepository { ConfigActivationListener configActivationListener, TenantListener tenantListener, ZookeeperServerConfig zookeeperServerConfig, - FileDirectory fileDirectory) { + FileDirectory fileDirectory, + OnnxModelCost onnxModelCost) { this(hostRegistry, curator, metrics, @@ -157,7 +160,8 @@ public class TenantRepository { configDefinitionRepo, configActivationListener, tenantListener, - zookeeperServerConfig); + zookeeperServerConfig, + onnxModelCost); } public TenantRepository(HostRegistry hostRegistry, @@ -178,7 +182,8 @@ public class TenantRepository { ConfigDefinitionRepo configDefinitionRepo, ConfigActivationListener configActivationListener, TenantListener tenantListener, - ZookeeperServerConfig zookeeperServerConfig) { + ZookeeperServerConfig zookeeperServerConfig, + OnnxModelCost onnxModelCost) { this.hostRegistry = hostRegistry; this.configserverConfig = configserverConfig; this.curator = curator; @@ -201,6 +206,7 @@ public class TenantRepository { this.zookeeperServerConfig = zookeeperServerConfig; // This we should control with a feature flag. this.deployHelperExecutor = createModelBuilderExecutor(); + this.onnxModelCost = onnxModelCost; curator.framework().getConnectionStateListenable().addListener(this::stateChanged); @@ -353,7 +359,8 @@ public class TenantRepository { curator, zone, flagSource, - secretStore); + secretStore, + onnxModelCost); SessionRepository sessionRepository = new SessionRepository(tenantName, applicationRepo, sessionPreparer, @@ -371,7 +378,8 @@ public class TenantRepository { clock, modelFactoryRegistry, configDefinitionRepo, - zookeeperServerConfig.juteMaxBuffer()); + zookeeperServerConfig.juteMaxBuffer(), + onnxModelCost); log.log(Level.FINE, "Adding tenant '" + tenantName + "'" + ", created " + created + ". Bootstrapping in " + Duration.between(start, clock.instant())); Tenant tenant = new Tenant(tenantName, sessionRepository, applicationRepo, created); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java index 4c262379c35..1288b63cadd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java @@ -111,6 +111,12 @@ public class ZKApplication { return getBytesInternal(getFullPath(path)); } + public long getSize(Path path) { + return curator.getStat(path).map(stat -> (long)stat.getDataLength()) + .orElseThrow(() -> new IllegalArgumentException( + "Could not get size from '" + path + "' in zookeeper")); + } + void putData(Path path, String data) { byte[] bytes = Utf8.toBytes(data); ensureDataIsNotTooLarge(bytes, path); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java index 6bc29331efb..e51f8627de2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java @@ -3,8 +3,9 @@ package com.yahoo.vespa.config.server.zookeeper; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.path.Path; import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.util.ConfigUtils; import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; @@ -13,11 +14,9 @@ import java.io.InputStream; import java.io.Reader; import java.io.StringReader; import java.io.StringWriter; -import java.util.logging.Level; -import com.yahoo.vespa.config.util.ConfigUtils; - import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH; @@ -184,6 +183,8 @@ class ZKApplicationFile extends ApplicationFile { } } + @Override public long getSize() { return zkApp.getSize(getZKPath(path)); } + @Override public int compareTo(ApplicationFile other) { if (other == this) return 0; diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 02481291213..a1e9bc3054b 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -26,6 +26,7 @@ <component id="com.yahoo.vespa.config.server.tenant.TenantRepository" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.host.HostRegistry" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.ApplicationRepository" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.FallbackOnnxModelCostProvider" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.HealthCheckerProviderProvider" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.version.VersionState" bundle="configserver" /> <component id="com.yahoo.config.provision.Zone" bundle="config-provisioning" /> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 104727cb4f3..333dae94769 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -605,7 +605,7 @@ public class ApplicationRepositoryTest { long sessionId = result.sessionId(); exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("Session is active: 2"); + exceptionRule.expectMessage("Session 2 for 'test1' is active"); applicationRepository.prepare(sessionId, prepareParams()); exceptionRule.expect(IllegalArgumentException.class); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index f5cd56707b3..fccb6785cb8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -78,6 +79,7 @@ public class ModelContextImplTest { Optional.empty(), List.of()), Optional.empty(), + OnnxModelCost.disabled(), Optional.empty(), new Version(7), new Version(8)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClientTest.java new file mode 100644 index 00000000000..03e379311cc --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClientTest.java @@ -0,0 +1,123 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application;// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.yahoo.config.ConfigInstance.Builder; +import com.yahoo.config.FileReference; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ApplicationClusterEndpoint.AuthMethod; +import com.yahoo.config.model.api.ApplicationClusterEndpoint.DnsName; +import com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod; +import com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope; +import com.yahoo.config.model.api.ApplicationClusterInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.PortInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.config.server.application.ActiveTokenFingerprints.Token; +import com.yahoo.vespa.config.server.modelfactory.ModelResult; +import org.junit.Rule; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.okJson; +import static com.github.tomakehurst.wiremock.client.WireMock.serverError; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; +import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class ActiveTokenFingerprintsClientTest { + + @Rule public final WireMockRule server1 = new WireMockRule(options().dynamicPort(), true); + @Rule public final WireMockRule server2 = new WireMockRule(options().dynamicPort(), true); + @Rule public final WireMockRule server3 = new WireMockRule(options().dynamicPort(), true); + @Rule public final WireMockRule server4 = new WireMockRule(options().dynamicPort(), true); + + @Test + public void verifyMultipleResponsesCombine() throws Exception { + try (ActiveTokenFingerprintsClient client = new ActiveTokenFingerprintsClient()) { + ModelResult app = MockModel::new; + String uriPath = "/data-plane-tokens/v1"; + server1.stubFor(get(urlEqualTo(uriPath)).willReturn(serverError())); + server2.stubFor(get(urlEqualTo(uriPath)).willReturn(okJson(""" + { "tokens": [ {"id": "t1", "fingerprints": [ "foo", "bar", "baz" ] } ] } + """))); + server3.stubFor(get(urlEqualTo(uriPath)).willReturn(aResponse().withStatus(503))); + server4.stubFor(get(urlEqualTo(uriPath)).willReturn(okJson(""" + { "tokens": [ {"id": "t2", "fingerprints": [ "quu" ] } ] } + """))); + Map<String, List<Token>> expected = Map.of("localhost", + List.of(new Token("t1", List.of("foo", "bar", "baz")))); + assertEquals(expected, client.get(app)); + } + } + + private class MockModel implements Model { + + @Override + public Collection<HostInfo> getHosts() { + return List.of(host(server1.port(), "localhost"), + host(server2.port(), "localhost"), + host(server3.port(), "localhost"), + host(server4.port(), "127.0.0.1")); // Should not be included, see application cluster info below. + + } + + private HostInfo host(int port, String host) { + return new HostInfo(host, + List.of(new ServiceInfo("container", + CONTAINER.serviceName, + List.of(new PortInfo(port, List.of("http"))), + Map.of(), + "myconfigId", + host), + new ServiceInfo("logserver", + LOGSERVER_CONTAINER.serviceName, + List.of(new PortInfo(port, List.of("http"))), + Map.of(), + "myconfigId", + "127.0.0.1"))); // Don't hit this. + } + + @Override + public Set<ApplicationClusterInfo> applicationClusterInfo() { + return Set.of(new ApplicationClusterInfo() { + @Override public List<ApplicationClusterEndpoint> endpoints() { + return List.of(ApplicationClusterEndpoint.builder() + .dnsName(DnsName.from("foo")) + .routingMethod(RoutingMethod.exclusive) + .authMethod(AuthMethod.token) + .scope(Scope.zone) + .clusterId("bar") + .hosts(List.of("localhost")) + .build()); + } + @Override public boolean getDeferChangesUntilRestart() { throw new UnsupportedOperationException(); } + @Override public String name() { throw new UnsupportedOperationException(); } + }); + } + + @Override public Builder getConfigInstance(ConfigKey<?> configKey, ConfigDefinition configDefinition) { throw new UnsupportedOperationException(); } + @Override public Set<ConfigKey<?>> allConfigsProduced() { throw new UnsupportedOperationException(); } + @Override public Set<String> allConfigIds() { throw new UnsupportedOperationException(); } + @Override public Set<FileReference> fileReferences() { throw new UnsupportedOperationException(); } + @Override public AllocatedHosts allocatedHosts() { throw new UnsupportedOperationException(); } + + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index 6bd20a29cf8..72cfe466993 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -10,6 +10,8 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Map; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * Base class for session handler tests * @@ -52,7 +54,7 @@ public class SessionHandlerTest { public static String getRenderedString(HttpResponse response) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); - return baos.toString(StandardCharsets.UTF_8); + return baos.toString(UTF_8); } public enum Cmd { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 951ef9df2f4..6fb5db70b68 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.config.server.MockLogRetriever; import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.MockSecretStoreValidator; import com.yahoo.vespa.config.server.MockTesterClient; +import com.yahoo.vespa.config.server.application.ActiveTokenFingerprints.Token; import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase; import com.yahoo.vespa.config.server.application.ApplicationReindexing; import com.yahoo.vespa.config.server.application.ClusterReindexing; @@ -117,6 +118,7 @@ public class ApplicationHandlerTest { private ManualClock clock; private List<Endpoint> expectedEndpoints; private Availability availability; + private Map<String, List<Token>> activeTokenFingerprints; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -140,6 +142,7 @@ public class ApplicationHandlerTest { .build(); tenantRepository.addTenant(mytenantName); orchestrator = new OrchestratorMock(); + activeTokenFingerprints = new HashMap<>(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withOrchestrator(orchestrator) @@ -149,6 +152,7 @@ public class ApplicationHandlerTest { .withConfigserverConfig(configserverConfig) .withSecretStoreValidator(secretStoreValidator) .withEndpointsChecker(endpoints -> { assertEquals(expectedEndpoints, endpoints); return availability; }) + .withActiveTokens(activeTokenFingerprints) .build(); } @@ -238,6 +242,19 @@ public class ApplicationHandlerTest { } @Test + public void testGetTokenFingerprints() throws IOException { + applicationRepository.deploy(testApp, prepareParams(applicationId)); + activeTokenFingerprints.putAll(Map.of("host", List.of(new Token("t1", List.of("fingers", "toes")), + new Token("t2", List.of())), + "toast", List.of())); + HttpResponse response = createApplicationHandler().handleGET(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/active-token-fingerprints", GET)); + assertEquals(200, response.getStatus()); + assertEquals(""" + {"hosts":[{"host":"host","tokens":[{"id":"t1","fingerprints":["fingers","toes"]},{"id":"t2","fingerprints":[]}]},{"host":"toast","tokens":[]}]}""", + getRenderedString(response)); + } + + @Test public void testReindex() throws Exception { ApplicationCuratorDatabase database = applicationRepository.getTenant(applicationId).getApplicationRepo().database(); reindexing(applicationId, GET, "{\"error-code\": \"NOT_FOUND\", \"message\": \"Application 'default.default' not found\"}", 404); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index 765523177a9..88aed6b058c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -92,7 +92,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void require_error_when_session_id_does_not_exist() throws Exception { // No session with this id exists HttpResponse response = request(HttpRequest.Method.PUT, 9999L); - assertHttpStatusCodeErrorCodeAndMessage(response, NOT_FOUND, HttpErrorResponse.ErrorCode.NOT_FOUND, "Session 9999 was not found"); + assertHttpStatusCodeErrorCodeAndMessage(response, NOT_FOUND, HttpErrorResponse.ErrorCode.NOT_FOUND, "Local session 9999 for 'test' was not found"); } @Test @@ -180,7 +180,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { HttpResponse getResponse = request(HttpRequest.Method.GET, 9999L); assertHttpStatusCodeErrorCodeAndMessage(getResponse, NOT_FOUND, HttpErrorResponse.ErrorCode.NOT_FOUND, - "Session 9999 was not found"); + "Remote session 9999 for 'test' was not found"); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index ca2f9da3273..8d1b0fefbaf 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -50,10 +50,11 @@ import static org.junit.Assert.assertTrue; public class LbServicesProducerTest { private static final Set<ContainerEndpoint> endpoints = Set.of( + new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.zone, List.of("mydisc.foo.foo.endpoint1.suffix")), + new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.zone, List.of("mydisc.foo.foo.endpoint2.suffix")), new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.global, List.of("rotation-1", "rotation-2")), new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.application, List.of("app-endpoint")) ); - private static final List<String> zoneDnsSuffixes = List.of(".endpoint1.suffix", ".endpoint2.suffix"); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); @@ -228,7 +229,7 @@ public class LbServicesProducerTest { private TestProperties getTestproperties(ApplicationId applicationId) { return new TestProperties() .setHostedVespa(true) - .setZoneDnsSuffixes(zoneDnsSuffixes) .setApplicationId(applicationId); } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 0158aa1961d..6dbb0d72c87 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; @@ -132,7 +133,8 @@ public class SessionPreparerTest { curator, zone, flagSource, - secretStore); + secretStore, + OnnxModelCost.disabled()); } @Test(expected = InvalidApplicationException.class) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 02ee3202475..1417df73cfc 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -6,6 +6,7 @@ import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.Version; import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.concurrent.StripedExecutor; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -230,7 +231,8 @@ public class TenantRepositoryTest { new TestConfigDefinitionRepo(), new TenantApplicationsTest.MockConfigActivationListener(), new MockTenantListener(), - new ZookeeperServerConfig.Builder().myid(0).build()); + new ZookeeperServerConfig.Builder().myid(0).build(), + OnnxModelCost.disabled()); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java index dd982ccbd72..0419a313dea 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java @@ -6,6 +6,7 @@ import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.ConfigServerDB; import com.yahoo.vespa.config.server.MockSecretStore; @@ -64,7 +65,8 @@ public class TestTenantRepository extends TenantRepository { configDefinitionRepo, configActivationListener, tenantListener, - new ZookeeperServerConfig.Builder().myid(0).build()); + new ZookeeperServerConfig.Builder().myid(0).build(), + OnnxModelCost.disabled()); } public static class Builder { diff --git a/container-disc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java b/container-disc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java index 67862533259..c29a4c1d009 100644 --- a/container-disc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java +++ b/container-disc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java @@ -15,6 +15,8 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.Container; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; +import com.yahoo.jdisc.Request; +import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.AbstractRequestHandler; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; @@ -82,22 +84,18 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { } @Override - public ContentChannel handleRequest(com.yahoo.jdisc.Request request, ResponseHandler handler) { - FastContentWriter writer = new FastContentWriter(new ResponseDispatch() { - @Override - protected com.yahoo.jdisc.Response newResponse() { - com.yahoo.jdisc.Response response = new com.yahoo.jdisc.Response(com.yahoo.jdisc.Response.Status.OK); + public ContentChannel handleRequest(Request request, ResponseHandler handler) { + try (FastContentWriter writer = new FastContentWriter(new ResponseDispatch() { + @Override protected Response newResponse() { + Response response = new Response(Response.Status.OK); response.headers().add("Content-Type", List.of("application/json")); return response; } - }.connect(handler)); - - try { + }.connect(handler))) { writer.write(jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(render())); } catch (JsonProcessingException e) { throw new RuntimeException("Invalid JSON: " + e.getMessage(), e); } - writer.close(); return new IgnoredContent(); } diff --git a/container-search/src/main/java/com/yahoo/search/logging/LoggerEntry.java b/container-search/src/main/java/com/yahoo/search/logging/LoggerEntry.java index 80ff967f779..d8e656d1b2b 100644 --- a/container-search/src/main/java/com/yahoo/search/logging/LoggerEntry.java +++ b/container-search/src/main/java/com/yahoo/search/logging/LoggerEntry.java @@ -135,7 +135,7 @@ public class LoggerEntry { return logger.send(new LoggerEntry(this)); } - LoggerEntry build() { + public LoggerEntry build() { return new LoggerEntry(this); } diff --git a/container-search/src/main/java/com/yahoo/search/logging/Spooler.java b/container-search/src/main/java/com/yahoo/search/logging/Spooler.java index 921b8f444f1..cf750fd9d8a 100644 --- a/container-search/src/main/java/com/yahoo/search/logging/Spooler.java +++ b/container-search/src/main/java/com/yahoo/search/logging/Spooler.java @@ -85,10 +85,10 @@ public class Spooler { public void processFiles(Function<LoggerEntry, Boolean> transport) throws IOException { List<Path> files = listFilesInPath(readyPath); if (files.size() == 0) { - log.log(Level.FINEST, "No files in ready path " + readyPath.toFile().getAbsolutePath()); + log.log(Level.FINEST, () -> "No files in ready path " + readyPath.toFile().getAbsolutePath()); return; } - log.log(Level.FINE, "Files in ready path: " + files.size()); + log.log(Level.FINE, () -> "Files in ready path: " + files.size()); List<File> fileList = getFiles(files); if ( ! fileList.isEmpty()) { @@ -116,7 +116,7 @@ public class Spooler { List<String> lines = Files.readAllLines(f.toPath()); for (String line : lines) { LoggerEntry entry = LoggerEntry.deserialize(line); - log.log(Level.FINE, "Read entry " + entry + " from " + f); + log.log(Level.FINE, () -> "Read entry " + entry + " from " + f); success = transport.apply(entry); if (! success) { throw new RuntimeException("Unable to process file " + f + ": unsuccessful call to transport() for " + entry); @@ -190,7 +190,7 @@ public class Spooler { String fileName = currentFileName(); Path file = spoolPath.resolve(processingPath).resolve(fileName); try { - log.log(Level.FINE, "Writing entry " + entryCounter.get() + " (" + entry.serialize() + ") to file " + fileName); + log.log(Level.FINE, () -> "Writing entry " + entryCounter.get() + " (" + entry.serialize() + ") to file " + fileName); Files.writeString(file, entry.serialize() + "\n", StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE); firstWriteTimestamp.compareAndExchange(Instant.EPOCH, clock.instant()); entryCounter.incrementAndGet(); @@ -242,7 +242,7 @@ public class Spooler { if (file.exists() && file.canRead() && file.canWrite()) { log.log(Level.INFO, "Directory " + path + " already exists"); } else if (file.mkdirs()) { - log.log(Level.FINE, "Created " + path); + log.log(Level.FINE, () -> "Created " + path); } else { log.log(Level.WARNING, "Could not create " + path + ", please check permissions"); } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java index 3edad64f9f2..3dfa278662d 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java @@ -135,14 +135,14 @@ public class NGramSearcher extends Searcher { * Creates the root of the query subtree which will contain the grams to match, * called by {@link #splitToGrams}. This hook is provided to make it easy to create a subclass which * matches grams using a different composite item, e.g an OrItem. - * <p> + * * This default implementation returns createGramRoot(query). * * @param term the term item this gram root is replacing in the query tree, * typically used to access the index name of the term when that is required by the new gram root * (such as in PhraseItem) * @param query the input query, to make it possible to return a different composite item type - * depending on the query content + * depending on the query content * @return the composite item to add the gram items to in {@link #splitToGrams} */ protected CompositeItem createGramRoot(HasIndexItem term, Query query) { diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/EquivTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/EquivTestCase.java new file mode 100644 index 00000000000..3e2c634b7f2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/EquivTestCase.java @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import org.junit.jupiter.api.Test; + +/** + * @author bratseth + */ +public class EquivTestCase extends RuleBaseAbstractTestCase { + + public EquivTestCase() { + super("equiv.sr"); + } + + @Test + void testEquiv() { + assertSemantics("EQUIV \"lord of the rings\" lotr", "lotr"); + } + + @Test + void testEquivWithFollowingQuery() { + assertSemantics("AND (EQUIV \"lord of the rings\" lotr) is a movie", "lotr is a movie"); + } + + @Test + void testEquivWithPrecedingQuery() { + assertSemantics("AND a movie is (EQUIV \"lord of the rings\" lotr)", "a movie is lotr"); + } + + @Test + void testEquivWithSurroundingQuery() { + assertSemantics("AND a movie is (EQUIV \"lord of the rings\" lotr) yes", "a movie is lotr yes"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java index 49449153d1f..4d25ebdccff 100644 --- a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java @@ -334,15 +334,15 @@ public class NGramSearcherTestCase { Result r = new Execution(new Chain<>(createSearcher(), new MockBackend1()), createContextStub(createIndexFacts())).search(q); Hit h1 = r.hits().get("hit1"); assertEquals("Should be untouched,\u001feven if containing \u001f", - h1.getField("test").toString()); + h1.getField("test").toString()); assertTrue(h1.getField("test") instanceof String); assertEquals("Blue red Ed A", h1.getField("gram2").toString()); assertTrue(h1.getField("gram2") instanceof XMLString); assertEquals("Blue red ed a\u001f", - h1.getField("gram3").toString(), - "Separators on borders work"); + h1.getField("gram3").toString(), + "Separators on borders work"); assertTrue(h1.getField("gram3") instanceof String); Hit h2 = r.hits().get("hit2"); @@ -352,7 +352,7 @@ public class NGramSearcherTestCase { Hit h3 = r.hits().get("hit3"); assertEquals("\u001ffin\u001f \u001fen\u001f \u001fa\u001f", h3.getField("gram2").toString()); assertEquals("#Logging in #Java is like that \"Judean P\u001fopul\u001far Front\" scene from \"Life of Brian\".", - h3.getField("gram3").toString()); + h3.getField("gram3").toString()); } private Item parseQuery(String query, Query.Type type) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java index ed965f4331e..66cf3eef954 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.archive; +import com.yahoo.component.Version; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; @@ -10,6 +11,7 @@ import java.net.URI; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Stream; /** * Service that manages archive storage URIs for tenant nodes. @@ -28,4 +30,19 @@ public interface ArchiveService { Optional<String> findEnclaveArchiveBucket(ZoneId zoneId, CloudAccount cloudAccount); URI bucketURI(ZoneId zoneId, String bucketName); + + /** + * @return the version of the template that was used during the last apply for the given cloud account, + * or {@link Version#emptyVersion} if the version tag was not present or invalid, + * or {@link Optional#empty()} if the we have no access to the cloud account (template probably not applied yet) + */ + Optional<Version> getEnclaveTemplateVersion(CloudAccount cloudAccount); + + static Stream<Version> parseVersion(String versionString) { + try { + return Stream.of(Version.fromString(versionString)); + } catch (IllegalArgumentException e) { + return Stream.empty(); + } + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java index 7461d3aa47e..4e6e71ca855 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.archive; +import com.yahoo.component.Version; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; @@ -59,6 +60,11 @@ public class MockArchiveService implements ArchiveService { return URI.create(String.format("s3://%s/", bucketName)); } + @Override + public Optional<Version> getEnclaveTemplateVersion(CloudAccount cloudAccount) { + return Optional.of(new Version(1, 2, 3)); + } + public void setEnclaveArchiveBucket(ZoneId zoneId, CloudAccount cloudAccount, String bucketName) { removeEnclaveArchiveBucket(zoneId, cloudAccount); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java index 5c6e5c9542a..dfa0d4dccf2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.billing; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import java.math.BigDecimal; import java.time.LocalDate; @@ -109,6 +110,9 @@ public interface BillingController { /** Get all bills from the system */ List<Bill> getBills(); + /** Get the bill with the given id */ + Bill getBill(Bill.Id billId); + /** Get the bill collection method for the given tenant */ default CollectionMethod getCollectionMethod(TenantName tenant) { return CollectionMethod.NONE; @@ -125,4 +129,8 @@ public interface BillingController { } default void updateCache(List<TenantName> tenants) {} -}
\ No newline at end of file + + default String exportBill(Bill bill, String exportMethod, CloudTenant tenant) { + return "NOT_IMPLEMENTED"; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporter.java index a4b3abf3bf9..719d22429b8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporter.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporter.java @@ -1,5 +1,8 @@ package com.yahoo.vespa.hosted.controller.api.integration.billing; +import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; + public interface BillingReporter { - double maintain(); + BillingReference maintainTenant(CloudTenant tenant); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporterMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporterMock.java new file mode 100644 index 00000000000..34599f83a8c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporterMock.java @@ -0,0 +1,21 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; + +import java.time.Clock; +import java.util.UUID; + +public class BillingReporterMock implements BillingReporter { + private final Clock clock; + + public BillingReporterMock(Clock clock) { + this.clock = clock; + } + + @Override + public BillingReference maintainTenant(CloudTenant tenant) { + return new BillingReference(UUID.randomUUID().toString(), clock.instant()); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java index 671739bacab..eb20126304e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java @@ -171,6 +171,15 @@ public class MockBillingController implements BillingController { } @Override + public Bill getBill(Bill.Id billId) { + return committedBills.values().stream() + .flatMap(Collection::stream) + .filter(bill -> bill.id().equals(billId)) + .findFirst() + .orElseThrow(); + } + + @Override public CollectionMethod getCollectionMethod(TenantName tenant) { return collectionMethod.getOrDefault(tenant, CollectionMethod.AUTO); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 19bfc84db7a..31fdc9d1b64 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.EndpointsChecker.Availability; import com.yahoo.config.provision.EndpointsChecker.Endpoint; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL.Path; @@ -16,6 +17,8 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus import com.yahoo.vespa.hosted.controller.api.application.v4.model.SearchNodeMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; +import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.FingerPrint; +import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; @@ -158,4 +161,7 @@ public interface ConfigServer { /** Validates secret store configuration. */ String validateSecretStore(DeploymentId deploymentId, TenantSecretStore tenantSecretStore, String region, String parameterName); + /** Fingerprints of active data plane tokens, per healthy host with token auth, in the given deployment. */ + Map<HostName, Map<TokenId, List<FingerPrint>>> activeTokenFingerprints(DeploymentId deploymentId); + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java index 39975138140..48cdc6ee053 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java @@ -29,7 +29,7 @@ public class MockVpcEndpointService implements VpcEndpointService { } @Override - public synchronized Optional<DnsChallenge> setPrivateDns(DomainName privateDnsName, ClusterId clusterId, Optional<CloudAccount> account) { + public synchronized Optional<DnsChallenge> setPrivateDns(DomainName privateDnsName, ClusterId clusterId, Optional<CloudAccount> account, boolean isGenerated) { DnsChallenge challenge = new DnsChallenge(RecordName.from("challenge--" + privateDnsName.value()), RecordData.from(account.map(CloudAccount::value).orElse("system")), clusterId, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java index a3ee7681e2a..97e1b88b25c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java @@ -38,7 +38,7 @@ public interface VpcEndpointService { enum ChallengeState { pending, ready, running, done } /** Sets the private DNS name for any VPC endpoint for the given cluster, potentially guarded by a challenge. */ - Optional<DnsChallenge> setPrivateDns(DomainName privateDnsName, ClusterId clusterId, Optional<CloudAccount> account); + Optional<DnsChallenge> setPrivateDns(DomainName privateDnsName, ClusterId clusterId, Optional<CloudAccount> account, boolean isGenerated); /** Attempts to complete the challenge, and returns the updated challenge state. */ ChallengeState process(DnsChallenge challenge); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index e661c88e117..8f47ac68cda 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterSpec; @@ -191,8 +192,8 @@ public class SystemFlagsDataArchive { flagData.rules().forEach(rule -> rule.conditions().forEach(condition -> { int force_switch_expression_dummy = switch (condition.type()) { case RELATIONAL -> switch (condition.dimension()) { - case INSTANCE_ID, CLOUD, CLOUD_ACCOUNT, CLUSTER_ID, CLUSTER_TYPE, CONSOLE_USER_EMAIL, - ENVIRONMENT, HOSTNAME, NODE_TYPE, SYSTEM, TENANT_ID, ZONE_ID -> + case APPLICATION_ID, CLOUD, CLOUD_ACCOUNT, CLUSTER_ID, CLUSTER_TYPE, CONSOLE_USER_EMAIL, + ENVIRONMENT, HOSTNAME, INSTANCE_ID, NODE_TYPE, SYSTEM, TENANT_ID, ZONE_ID -> throw new FlagValidationException(condition.type().toWire() + " " + DimensionHelper.toWire(condition.dimension()) + " condition is not supported"); @@ -206,7 +207,7 @@ public class SystemFlagsDataArchive { }; case WHITELIST, BLACKLIST -> switch (condition.dimension()) { - case INSTANCE_ID -> validateConditionValues(condition, ApplicationId::fromSerializedForm); + case APPLICATION_ID -> validateConditionValues(condition, SystemFlagsDataArchive::validateTenantApplication); case CONSOLE_USER_EMAIL -> validateConditionValues(condition, email -> { if (!email.contains("@")) throw new FlagValidationException("Invalid email address: " + email); @@ -220,6 +221,7 @@ public class SystemFlagsDataArchive { case CLUSTER_TYPE -> validateConditionValues(condition, ClusterSpec.Type::from); case ENVIRONMENT -> validateConditionValues(condition, Environment::from); case HOSTNAME -> validateConditionValues(condition, HostName::of); + case INSTANCE_ID -> validateConditionValues(condition, ApplicationId::fromSerializedForm); case NODE_TYPE -> validateConditionValues(condition, NodeType::valueOf); case SYSTEM -> throw new IllegalStateException("Flag data contains system dimension"); case TENANT_ID -> validateConditionValues(condition, TenantName::from); @@ -250,23 +252,20 @@ public class SystemFlagsDataArchive { return 0; // dummy to force switch expression } + private static void validateTenantApplication(String application) { + String[] parts = application.split(":"); + if (parts.length != 2) + throw new IllegalArgumentException("Applications must be on the form tenant:application, but was %s".formatted(application)); + TenantName.from(parts[0]); + ApplicationName.from(parts[1]); + } + private static FlagData parseFlagData(FlagId flagId, String fileContent, ZoneRegistry zoneRegistry, boolean inController) { if (fileContent.isBlank()) return new FlagData(flagId); final JsonNode root; try { root = mapper.readTree(fileContent); - // TODO (mortent): Remove this after completing migration of APPLICATION_ID dimension - // replace "application" with "instance" for all dimension fields -// List<JsonNode> dimensionParents = root.findParents("dimension"); -// for (JsonNode parentNode : dimensionParents) { -// JsonNode dimension = parentNode.get("dimension"); -// if (dimension.isTextual() && "application".equals(dimension.textValue())) { -// ObjectNode parent = (ObjectNode) parentNode; -// parent.remove("dimension"); -// parent.put("dimension", "instance"); -// } -// } } catch (JsonProcessingException e) { throw new FlagValidationException("Invalid JSON: " + e.getMessage()); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java index 53a3f431de7..0754a5ed49f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import java.time.Instant; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -27,8 +28,9 @@ public class AthenzTenant extends Tenant { * Use {@link #create(TenantName, AthenzDomain, Property, Optional, Instant)}. * */ public AthenzTenant(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, - Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained) { - super(name, createdAt, lastLoginInfo, contact, tenantRolesLastMaintained); + Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained, + List<CloudAccountInfo> cloudAccounts) { + super(name, createdAt, lastLoginInfo, contact, tenantRolesLastMaintained, cloudAccounts); this.domain = Objects.requireNonNull(domain, "domain must be non-null"); this.property = Objects.requireNonNull(property, "property must be non-null"); this.propertyId = Objects.requireNonNull(propertyId, "propertyId must be non-null"); @@ -62,7 +64,7 @@ public class AthenzTenant extends Tenant { /** Create a new Athenz tenant */ public static AthenzTenant create(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, Instant createdAt) { - return new AthenzTenant(requireName(name), domain, property, propertyId, Optional.empty(), createdAt, LastLoginInfo.EMPTY, Instant.EPOCH); + return new AthenzTenant(requireName(name), domain, property, propertyId, Optional.empty(), createdAt, LastLoginInfo.EMPTY, Instant.EPOCH, List.of()); } @Override diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudAccountInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudAccountInfo.java new file mode 100644 index 00000000000..430f5770165 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudAccountInfo.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.tenant; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudAccount; + +import java.util.Objects; + +/** + * @author freva + */ +public record CloudAccountInfo(CloudAccount cloudAccount, Version templateVersion) { + + public CloudAccountInfo { + Objects.requireNonNull(cloudAccount, "cloudAccount must be non-null"); + Objects.requireNonNull(templateVersion, "templateVersion must be non-null"); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index 4d7aee7b604..173d3e1950e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java @@ -34,8 +34,8 @@ public class CloudTenant extends Tenant { BiMap<PublicKey, SimplePrincipal> developerKeys, TenantInfo info, List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, Optional<Instant> invalidateUserSessionsBefore, Instant tenantRoleLastMaintained, - Optional<BillingReference> billingReference) { - super(name, createdAt, lastLoginInfo, Optional.empty(), tenantRoleLastMaintained); + List<CloudAccountInfo> cloudAccounts, Optional<BillingReference> billingReference) { + super(name, createdAt, lastLoginInfo, Optional.empty(), tenantRoleLastMaintained, cloudAccounts); this.creator = creator; this.developerKeys = developerKeys; this.info = Objects.requireNonNull(info); @@ -51,7 +51,8 @@ public class CloudTenant extends Tenant { createdAt, LastLoginInfo.EMPTY, Optional.ofNullable(creator).map(SimplePrincipal::of), - ImmutableBiMap.of(), TenantInfo.empty(), List.of(), new ArchiveAccess(), Optional.empty(), Instant.EPOCH, Optional.empty()); + ImmutableBiMap.of(), TenantInfo.empty(), List.of(), new ArchiveAccess(), Optional.empty(), + Instant.EPOCH, List.of(), Optional.empty()); } /** The user that created the tenant */ diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/DeletedTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/DeletedTenant.java index b58fdf81278..30ce5d5a3b2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/DeletedTenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/DeletedTenant.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.tenant; import com.yahoo.config.provision.TenantName; import java.time.Instant; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -17,7 +18,7 @@ public class DeletedTenant extends Tenant { private final Instant deletedAt; public DeletedTenant(TenantName name, Instant createdAt, Instant deletedAt) { - super(name, createdAt, LastLoginInfo.EMPTY, Optional.empty(), Instant.EPOCH); + super(name, createdAt, LastLoginInfo.EMPTY, Optional.empty(), Instant.EPOCH, List.of()); this.deletedAt = Objects.requireNonNull(deletedAt, "deletedAt must be non-null"); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java index a4500991bf2..8b1c6b3ebde 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import java.time.Instant; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -20,13 +21,15 @@ public abstract class Tenant { private final LastLoginInfo lastLoginInfo; private final Optional<Contact> contact; private final Instant tenantRolesLastMaintained; + private final List<CloudAccountInfo> cloudAccounts; - Tenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Contact> contact, Instant tenantRolesLastMaintained) { + Tenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Contact> contact, Instant tenantRolesLastMaintained, List<CloudAccountInfo> cloudAccounts) { this.name = name; this.createdAt = createdAt; this.lastLoginInfo = lastLoginInfo; this.contact = contact; this.tenantRolesLastMaintained = tenantRolesLastMaintained; + this.cloudAccounts = cloudAccounts; } /** Name of this tenant */ @@ -53,6 +56,10 @@ public abstract class Tenant { return tenantRolesLastMaintained; } + public List<CloudAccountInfo> cloudAccounts() { + return cloudAccounts; + } + public abstract Type type(); @Override diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index 0b54152d058..6d7dc3a179e 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -246,11 +246,21 @@ public class SystemFlagsDataArchiveTest { "conditions": [ { "type": "whitelist", - "dimension": "application", + "dimension": "instance", "values": [ "f:o:o" ] } ], "value": true + }, + { + "conditions": [ + { + "type": "whitelist", + "dimension": "application", + "values": [ "f:o" ] + } + ], + "value": true } ] }""", @@ -288,11 +298,21 @@ public class SystemFlagsDataArchiveTest { { "comment": "bar", "type": "whitelist", - "dimension": "application", + "dimension": "instance", "values": [ "f:o:o" ] } ], "value": true + }, + { + "conditions": [ + { + "type": "whitelist", + "dimension": "application", + "values": [ "f:o" ] + } + ], + "value": true } ] }"""))); @@ -308,8 +328,7 @@ public class SystemFlagsDataArchiveTest { @Test void normalize_json_succeed_on_valid_values() { - addFile(Condition.Type.WHITELIST, "application", "a:b:c"); -// addFile(Condition.Type.WHITELIST, "instance", "a:b:c"); + addFile(Condition.Type.WHITELIST, "application", "a:b"); addFile(Condition.Type.WHITELIST, "cloud", "yahoo"); addFile(Condition.Type.WHITELIST, "cloud", "aws"); addFile(Condition.Type.WHITELIST, "cloud", "gcp"); @@ -322,6 +341,7 @@ public class SystemFlagsDataArchiveTest { addFile(Condition.Type.WHITELIST, "environment", "staging"); addFile(Condition.Type.WHITELIST, "environment", "test"); addFile(Condition.Type.WHITELIST, "hostname", "2080046-v6-11.ostk.bm2.prod.gq1.yahoo.com"); + addFile(Condition.Type.WHITELIST, "instance", "a:b:c"); addFile(Condition.Type.WHITELIST, "node-type", "tenant"); addFile(Condition.Type.WHITELIST, "node-type", "host"); addFile(Condition.Type.WHITELIST, "node-type", "config"); @@ -363,12 +383,13 @@ public class SystemFlagsDataArchiveTest { @Test void normalize_json_fail_on_invalid_values() { - failAddFile(Condition.Type.WHITELIST, "application", "a.b.c", "In file flags/temporary/foo/default.json: Invalid application 'a.b.c' in whitelist condition: Application ids must be on the form tenant:application:instance, but was a.b.c"); + failAddFile(Condition.Type.WHITELIST, "application", "a.b", "In file flags/temporary/foo/default.json: Invalid application 'a.b' in whitelist condition: Applications must be on the form tenant:application, but was a.b"); failAddFile(Condition.Type.WHITELIST, "cloud", "foo", "In file flags/temporary/foo/default.json: Unknown cloud: foo"); // cluster-id: any String is valid failAddFile(Condition.Type.WHITELIST, "cluster-type", "foo", "In file flags/temporary/foo/default.json: Invalid cluster-type 'foo' in whitelist condition: Illegal cluster type 'foo'"); failAddFile(Condition.Type.WHITELIST, "console-user-email", "not-valid-email-address", "In file flags/temporary/foo/default.json: Invalid email address: not-valid-email-address"); failAddFile(Condition.Type.WHITELIST, "environment", "foo", "In file flags/temporary/foo/default.json: Invalid environment 'foo' in whitelist condition: 'foo' is not a valid environment identifier"); + failAddFile(Condition.Type.WHITELIST, "instance", "a.b.c", "In file flags/temporary/foo/default.json: Invalid instance 'a.b.c' in whitelist condition: Application ids must be on the form tenant:application:instance, but was a.b.c"); failAddFile(Condition.Type.WHITELIST, "hostname", "not:a:hostname", "In file flags/temporary/foo/default.json: Invalid hostname 'not:a:hostname' in whitelist condition: hostname must match '(([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9])\\.?', but got: 'not:a:hostname'"); failAddFile(Condition.Type.WHITELIST, "node-type", "footype", "In file flags/temporary/foo/default.json: Invalid node-type 'footype' in whitelist condition: No enum constant com.yahoo.config.provision.NodeType.footype"); failAddFile(Condition.Type.WHITELIST, "system", "bar", "In file flags/temporary/foo/default.json: Invalid system 'bar' in whitelist condition: 'bar' is not a valid system"); diff --git a/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-east-3.json b/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-east-3.json index b79e0913c22..c4dca9aa2e1 100644 --- a/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-east-3.json +++ b/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-east-3.json @@ -5,7 +5,7 @@ "conditions": [ { "type": "whitelist", - "dimension": "application", + "dimension": "instance", "values": ["a:b:c"] } ] diff --git a/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-west-1.json b/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-west-1.json index 75cffdea009..283b09d5c0b 100644 --- a/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-west-1.json +++ b/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-west-1.json @@ -5,7 +5,7 @@ "conditions": [ { "type": "whitelist", - "dimension": "application", + "dimension": "instance", "values": ["a:b:c"] } ], diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index 6ec732a3815..7d19acfce80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; @@ -43,12 +44,14 @@ public abstract class LockedTenant { final Instant createdAt; final LastLoginInfo lastLoginInfo; final Instant tenantRolesLastMaintained; + final List<CloudAccountInfo> cloudAccounts; - private LockedTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained) { + private LockedTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained, List<CloudAccountInfo> cloudAccounts) { this.name = requireNonNull(name); this.createdAt = requireNonNull(createdAt); this.lastLoginInfo = requireNonNull(lastLoginInfo); this.tenantRolesLastMaintained = requireNonNull(tenantRolesLastMaintained); + this.cloudAccounts = requireNonNull(cloudAccounts); } static LockedTenant of(Tenant tenant, Mutex lock) { @@ -66,6 +69,8 @@ public abstract class LockedTenant { public abstract LockedTenant with(Instant tenantRolesLastMaintained); + public abstract LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts); + public Deleted deleted(Instant deletedAt) { return new Deleted(new DeletedTenant(name, createdAt, deletedAt)); } @@ -85,8 +90,8 @@ public abstract class LockedTenant { private final Optional<Contact> contact; private Athenz(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, - Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained) { - super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained); + Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo, Instant tenantRolesLastMaintained, List<CloudAccountInfo> cloudAccounts) { + super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); this.domain = domain; this.property = property; this.propertyId = propertyId; @@ -94,38 +99,43 @@ public abstract class LockedTenant { } private Athenz(AthenzTenant tenant) { - this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.tenantRolesLastMaintained()); + this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.tenantRolesLastMaintained(), tenant.cloudAccounts()); } @Override public AthenzTenant get() { - return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } public Athenz with(AthenzDomain domain) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } public Athenz with(Property property) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } public Athenz with(PropertyId propertyId) { - return new Athenz(name, domain, property, Optional.of(propertyId), contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, Optional.of(propertyId), contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } public Athenz with(Contact contact) { - return new Athenz(name, domain, property, propertyId, Optional.of(contact), createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, Optional.of(contact), createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } @Override public LockedTenant with(LastLoginInfo lastLoginInfo) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } @Override public LockedTenant with(Instant tenantRolesLastMaintained) { - return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); + } + + @Override + public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); } } @@ -146,8 +156,8 @@ public abstract class LockedTenant { BiMap<PublicKey, SimplePrincipal> developerKeys, TenantInfo info, List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, Optional<Instant> invalidateUserSessionsBefore, Instant tenantRolesLastMaintained, - Optional<BillingReference> billingReference) { - super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained); + List<CloudAccountInfo> cloudAccounts, Optional<BillingReference> billingReference) { + super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); this.developerKeys = ImmutableBiMap.copyOf(developerKeys); this.creator = creator; this.info = info; @@ -158,12 +168,12 @@ public abstract class LockedTenant { } private Cloud(CloudTenant tenant) { - this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess(), tenant.invalidateUserSessionsBefore(), tenant.tenantRolesLastMaintained(), tenant.billingReference()); + this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess(), tenant.invalidateUserSessionsBefore(), tenant.tenantRolesLastMaintained(), tenant.cloudAccounts(), tenant.billingReference()); } @Override public CloudTenant get() { - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withDeveloperKey(PublicKey key, Principal principal) { @@ -174,51 +184,56 @@ public abstract class LockedTenant { if (keys.inverse().containsKey(simplePrincipal)) throw new IllegalArgumentException(principal + " is already associated with key " + KeyUtils.toPem(keys.inverse().get(simplePrincipal))); keys.put(key, simplePrincipal); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withoutDeveloperKey(PublicKey key) { BiMap<PublicKey, SimplePrincipal> keys = HashBiMap.create(developerKeys); keys.remove(key); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withInfo(TenantInfo newInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } @Override public LockedTenant with(LastLoginInfo lastLoginInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withSecretStore(TenantSecretStore tenantSecretStore) { ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); secretStores.add(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withoutSecretStore(TenantSecretStore tenantSecretStore) { ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); secretStores.remove(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withArchiveAccess(ArchiveAccess archiveAccess) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore,tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore,tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud withInvalidateUserSessionsBefore(Instant invalidateUserSessionsBefore) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, Optional.of(invalidateUserSessionsBefore), tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, Optional.of(invalidateUserSessionsBefore), tenantRolesLastMaintained, cloudAccounts, billingReference); } @Override public LockedTenant with(Instant tenantRolesLastMaintained) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); + } + + @Override + public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); } public Cloud with(BillingReference billingReference) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, Optional.of(billingReference)); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, Optional.of(billingReference)); } } @@ -229,7 +244,7 @@ public abstract class LockedTenant { private final Instant deletedAt; private Deleted(DeletedTenant tenant) { - super(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Instant.EPOCH); + super(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Instant.EPOCH, List.of()); this.deletedAt = tenant.deletedAt(); } @@ -247,6 +262,11 @@ public abstract class LockedTenant { public LockedTenant with(Instant tenantRolesLastMaintained) { return this; } + + @Override + public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { + return this; + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 091836a1eea..b1ffce65852 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -82,7 +82,8 @@ public class RoutingController { private final Controller controller; private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; - private final BooleanFlag randomizedEndpoints; + private final BooleanFlag generatedEndpoints; + private final BooleanFlag legacyEndpoints; public RoutingController(Controller controller, RotationsConfig rotationsConfig) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); @@ -90,7 +91,8 @@ public class RoutingController { this.rotationRepository = new RotationRepository(Objects.requireNonNull(rotationsConfig, "rotationsConfig must be non-null"), controller.applications(), controller.curator()); - this.randomizedEndpoints = Flags.RANDOMIZED_ENDPOINT_NAMES.bindTo(controller.flagSource()); + this.generatedEndpoints = Flags.RANDOMIZED_ENDPOINT_NAMES.bindTo(controller.flagSource()); + this.legacyEndpoints = Flags.LEGACY_ENDPOINTS.bindTo(controller.flagSource()); } /** Create a routing context for given deployment */ @@ -228,13 +230,14 @@ public class RoutingController { .in(controller.system())); // Only a single region endpoint is needed, not one per auth method if (isProduction && generatedEndpoint.authMethod() == AuthMethod.mtls) { - endpoints.add(regionEndpoint.generatedFrom(generatedEndpoint) + GeneratedEndpoint weightedGeneratedEndpoint = generatedEndpoint.withClusterPart(weightedClusterPart(cluster, deployment)); + endpoints.add(regionEndpoint.generatedFrom(weightedGeneratedEndpoint) .authMethod(AuthMethod.none) .in(controller.system())); } } } - return EndpointList.copyOf(endpoints); + return filterEndpoints(deployment.applicationId(), EndpointList.copyOf(endpoints)); } /** Read routing policies and return zone- and region-scoped endpoints for given deployment */ @@ -268,7 +271,7 @@ public class RoutingController { endpoints.add(builder.generatedFrom(ge).authMethod(ge.authMethod()).in(controller.system())); } } - return EndpointList.copyOf(endpoints); + return filterEndpoints(routingId.instance(), EndpointList.copyOf(endpoints)); } /** Returns application endpoints pointing to given deployments */ @@ -424,6 +427,13 @@ public class RoutingController { Optional.of(application.id()))); } + private EndpointList filterEndpoints(ApplicationId instance, EndpointList endpoints) { + if (generatedEndpointsEnabled(instance) && !legacyEndpointsEnabled(instance)) { + return endpoints.generated(); + } + return endpoints; + } + private void registerRotationEndpointsInDns(PreparedEndpoints prepared) { TenantAndApplicationId owner = TenantAndApplicationId.from(prepared.deployment().applicationId()); EndpointList globalEndpoints = prepared.endpoints().scope(Scope.global); @@ -476,6 +486,22 @@ public class RoutingController { .toList(); } + /** Generate the cluster part of a {@link GeneratedEndpoint} for use in a {@link Endpoint.Scope#weighted} endpoint */ + private String weightedClusterPart(ClusterSpec.Id cluster, DeploymentId deployment) { + // This ID must be common for a given cluster in all deployments within the same cloud-native region + String cloudNativeRegion = controller.zoneRegistry().zones().all().get(deployment.zoneId()).get().getCloudNativeRegionName(); + HashCode hash = Hashing.sha256().newHasher() + .putString(cluster.value(), StandardCharsets.UTF_8) + .putString(":", StandardCharsets.UTF_8) + .putString(cloudNativeRegion, StandardCharsets.UTF_8) + .putString(":", StandardCharsets.UTF_8) + .putString(deployment.applicationId().serializedForm(), StandardCharsets.UTF_8) + .hash(); + String alphabet = "abcdef"; + char letter = alphabet.charAt(Math.abs(hash.asInt()) % alphabet.length()); + return letter + hash.toString().substring(0, 7); + } + /** Returns existing generated endpoints, grouped by their {@link Scope#multiDeployment()} endpoint */ private Map<EndpointId, GeneratedEndpointList> readDeclaredGeneratedEndpoints(TenantAndApplicationId application) { Map<EndpointId, GeneratedEndpointList> endpoints = new HashMap<>(); @@ -525,7 +551,17 @@ public class RoutingController { } public boolean generatedEndpointsEnabled(ApplicationId instance) { - return randomizedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()).value(); + return generatedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) + .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) + .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) + .value(); + } + + public boolean legacyEndpointsEnabled(ApplicationId instance) { + return legacyEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) + .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) + .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) + .value(); } private static void requireGeneratedEndpoints(GeneratedEndpointList generatedEndpoints, boolean declared) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index bf2f2ab90eb..d11540b28dd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -165,6 +166,14 @@ public class TenantController { } } + public void updateCloudAccounts(TenantName tenantName, List<CloudAccountInfo> cloudAccounts) { + try (Mutex lock = lock(tenantName)) { + var tenant = require(tenantName); + if (tenant.cloudAccounts().equals(cloudAccounts)) return; // no change + curator.writeTenant(LockedTenant.of(tenant, lock).withCloudAccounts(cloudAccounts).get()); + } + } + /** Deletes the given tenant. */ public void delete(TenantName tenant, Optional<Credentials> credentials, boolean forget) { try (Mutex lock = lock(tenant)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java index 8db4492356a..28f9963f24c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java @@ -38,6 +38,11 @@ public record GeneratedEndpoint(String clusterPart, String applicationPart, Auth return !declared(); } + /** Returns a copy of this with cluster part set to given value */ + public GeneratedEndpoint withClusterPart(String clusterPart) { + return new GeneratedEndpoint(clusterPart, applicationPart, authMethod, endpoint); + } + /** Create a new endpoint part, using random as a source of randomness */ public static String createPart(RandomGenerator random) { String alphabet = "abcdef0123456789"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index e01da00a27e..33af58a9790 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -11,6 +11,7 @@ import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; @@ -57,6 +58,7 @@ public class EndpointCertificates { private final EndpointCertificateValidator certificateValidator; private final BooleanFlag useAlternateCertProvider; private final StringFlag endpointCertificateAlgo; + private final BooleanFlag assignLegacyNames; private final static Duration GCP_CERTIFICATE_EXPIRY_TIME = Duration.ofDays(100); // 100 days, 10 more than notAfter time public EndpointCertificates(Controller controller, EndpointCertificateProvider certificateProvider, @@ -64,6 +66,7 @@ public class EndpointCertificates { this.controller = controller; this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); + this.assignLegacyNames = Flags.LEGACY_ENDPOINTS.bindTo(controller.flagSource()); this.curator = controller.curator(); this.clock = controller.clock(); this.certificateProvider = certificateProvider; @@ -140,10 +143,11 @@ public class EndpointCertificates { } try (NestedTransaction transaction = new NestedTransaction()) { curator.removeUnassignedCertificate(candidate.get(), transaction); - curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, candidate.get().certificate()), + EndpointCertificate certificate = candidate.get().certificate().withLastRequested(clock.instant().getEpochSecond()); + curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, certificate), transaction); transaction.commit(); - return candidate.get().certificate(); + return certificate; } } } @@ -174,9 +178,12 @@ public class EndpointCertificates { } // Re-provision certificate if it is missing SANs for the zone we are deploying to - // Skip this validation for now if the cert has a randomized id + // Skip this validation for now if the cert has a randomized id and should not provision legacy names Optional<EndpointCertificate> currentCertificate = assignedCertificate.map(AssignedCertificate::certificate); - var requiredSansForZone = currentCertificate.get().randomizedId().isEmpty() ? + boolean legacyNames = assignLegacyNames.with(FetchVector.Dimension.INSTANCE_ID, instance.id().serializedForm()) + .with(FetchVector.Dimension.APPLICATION_ID, instance.id().toSerializedFormWithoutInstance()).value(); + + var requiredSansForZone = legacyNames || currentCertificate.get().randomizedId().isEmpty() ? controller.routing().certificateDnsNames(deployment, deploymentSpec) : List.<String>of(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index e247d6baa09..1b40781fe0f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -276,7 +276,7 @@ public class DeploymentTrigger { List<RetriggerEntry> retriggerEntries = controller.curator().readRetriggerEntries(); List<RetriggerEntry> newList = new ArrayList<>(retriggerEntries); RetriggerEntry requiredEntry = new RetriggerEntry(new JobId(deployment.applicationId(), jobType), run.id().number() + 1); - if(newList.stream().noneMatch(entry -> entry.jobId().equals(requiredEntry.jobId()) && entry.requiredRun()>=requiredEntry.requiredRun())) { + if (newList.stream().noneMatch(entry -> entry.jobId().equals(requiredEntry.jobId()) && entry.requiredRun() >= requiredEntry.requiredRun())) { newList.add(requiredEntry); } newList = newList.stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 919facee0c1..11c47d8f481 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -62,7 +63,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -362,21 +362,24 @@ public class InternalStepRunner implements StepRunner { Version platform = setTheStage ? versions.sourcePlatform().orElse(versions.targetPlatform()) : versions.targetPlatform(); Run run = controller.jobController().run(id); - Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone()), - Optional.of(platform)); + // In manually deployed zones it is allowed for some model versions not being built (e.g due to incompatibility) + // but deployment still succeeding, so we cannot use version when checking for config convergence + Optional<Version> platformVersion = id.type().environment().isManuallyDeployed() ? Optional.empty() : Optional.of(platform); + Optional<ServiceConvergence> services = configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone()), + platformVersion); if (services.isEmpty()) { logger.log("Config status not currently available -- will retry."); return Optional.empty(); } - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), - NodeFilter.all() - .applications(id.application()) - .states(active)); + List<Node> nodes = configServer().nodeRepository().list(id.type().zone(), + NodeFilter.all() + .applications(id.application()) + .states(active)); Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); - List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), - NodeFilter.all() - .hostnames(parentHostnames)); + List<Node> parents = configServer().nodeRepository().list(id.type().zone(), + NodeFilter.all() + .hostnames(parentHostnames)); boolean firstTick = run.convergenceSummary().isEmpty(); NodeList nodeList = NodeList.of(nodes, parents, services.get()); ConvergenceSummary summary = nodeList.summary(); @@ -496,8 +499,8 @@ public class InternalStepRunner implements StepRunner { ZoneId zone = id.type().zone(); ApplicationId testerId = id.tester().id(); - Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(testerId, zone), - Optional.of(platform)); + Optional<ServiceConvergence> services = configServer().serviceConvergence(new DeploymentId(testerId, zone), + Optional.of(platform)); if (services.isEmpty()) { if (run.stepInfo(installTester).get().startTime().get().isBefore(controller.clock().instant().minus(Duration.ofMinutes(30)))) { logger.log(WARNING, "Config status not available after 30 minutes; giving up!"); @@ -508,14 +511,14 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); } } - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(zone, - NodeFilter.all() - .applications(testerId) - .states(active, reserved)); + List<Node> nodes = configServer().nodeRepository().list(zone, + NodeFilter.all() + .applications(testerId) + .states(active, reserved)); Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); - List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(zone, - NodeFilter.all() - .hostnames(parentHostnames)); + List<Node> parents = configServer().nodeRepository().list(zone, + NodeFilter.all() + .hostnames(parentHostnames)); NodeList nodeList = NodeList.of(nodes, parents, services.get()); logger.log(nodeList.asList().stream() .flatMap(node -> nodeDetails(node, false)) @@ -534,6 +537,8 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); } + private ConfigServer configServer() { return controller.serviceRegistry().configServer(); } + /** Returns true iff all containers in the tester deployment give 100 consecutive 200 OK responses on /status.html. */ private boolean testerContainersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { DeploymentId deploymentId = new DeploymentId(id, zoneId); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java index d10e38fd990..e7ec6675a82 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java @@ -2,23 +2,76 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.LockedTenant; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporter; +import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; +import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; public class BillingReportMaintainer extends ControllerMaintainer { private final BillingReporter reporter; + private final BillingController billing; + private final PlanRegistry plans; public BillingReportMaintainer(Controller controller, Duration interval) { super(controller, interval, null, Set.of(SystemName.PublicCd)); this.reporter = controller.serviceRegistry().billingReporter(); + this.billing = controller.serviceRegistry().billingController(); + this.plans = controller.serviceRegistry().planRegistry(); } @Override protected double maintain() { - return this.reporter.maintain(); + maintainTenants(); + return 0.0; + } + + private void maintainTenants() { + var tenants = cloudTenants(); + var tenantNames = List.copyOf(tenants.keySet()); + var billableTenants = billableTenants(tenantNames); + + billableTenants.forEach(tenant -> { + controller().tenants().lockIfPresent(tenant, LockedTenant.Cloud.class, locked -> { + var ref = reporter.maintainTenant(locked.get()); + if (locked.get().billingReference().isEmpty() || ! locked.get().billingReference().get().equals(ref)) { + controller().tenants().store(locked.with(ref)); + } + }); + }); + } + + private Map<TenantName, CloudTenant> cloudTenants() { + return controller().tenants().asList() + .stream() + .filter(CloudTenant.class::isInstance) + .map(CloudTenant.class::cast) + .collect(Collectors.toMap( + Tenant::name, + Function.identity())); + } + + private List<Plan> billablePlans() { + return plans.all().stream() + .filter(Plan::isBilled) + .toList(); + } + + private List<TenantName> billableTenants(List<TenantName> tenants) { + return billablePlans().stream() + .flatMap(p -> billing.tenantsWithPlan(tenants, p.id()).stream()) + .toList(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java index 70eeb2b9f6c..ed383175cc3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java @@ -69,7 +69,7 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { // Create metric for available certificates in the pool as a fraction of configured size int poolSize = certPoolSize.value(); long available = certificatePool.stream().filter(c -> c.state() == UnassignedCertificate.State.ready).count(); - metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? (available/poolSize) : 1.0), metric.createContext(Map.of())); + metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? ((double)available/poolSize) : 1.0), metric.createContext(Map.of())); if (certificatePool.size() < poolSize) { provisionRandomizedCertificate(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java new file mode 100644 index 00000000000..f0fc8985bdf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudAccountVerifier.java @@ -0,0 +1,55 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; + +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; + +import static java.util.logging.Level.WARNING; + +/** + * Verifies the cloud accounts that may be used by a given user have applied the enclave template + * and extracts the version of the applied template. + * + * All maintainers that operate on external cloud accounts should use the list on the Tenant instance + * maintained by this class rather than the cloud-accounts feature flag. + * + * The template version can be used to determine if new features can be enabled for the cloud account. + * + * @author freva + */ +public class CloudAccountVerifier extends ControllerMaintainer { + + private static final Logger logger = Logger.getLogger(CloudAccountVerifier.class.getName()); + + CloudAccountVerifier(Controller controller, Duration interval) { + super(controller, interval, null, Set.of(SystemName.PublicCd, SystemName.Public)); + } + + @Override + protected double maintain() { + int attempts = 0, failures = 0; + for (Tenant tenant : controller().tenants().asList()) { + try { + attempts++; + List<CloudAccountInfo> cloudAccountInfos = controller().applications().accountsOf(tenant.name()).stream() + .flatMap(account -> controller().serviceRegistry() + .archiveService() + .getEnclaveTemplateVersion(account) + .map(version -> new CloudAccountInfo(account, version)) + .stream()) + .toList(); + controller().tenants().updateCloudAccounts(tenant.name(), cloudAccountInfos); + } catch (RuntimeException e) { + logger.log(WARNING, "Failed to verify cloud accounts for tenant " + tenant.name(), e); + failures++; + } + } + return asSuccessFactorDeviation(attempts, failures); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index f9c93a87c44..f6da3609fbb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -29,8 +29,8 @@ public class ContactInformationMaintainer extends ControllerMaintainer { private final ContactRetriever contactRetriever; - public ContactInformationMaintainer(Controller controller, Duration interval) { - super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); + public ContactInformationMaintainer(Controller controller, Duration interval, Double successFactorBaseline) { + super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic)), successFactorBaseline); this.contactRetriever = controller.serviceRegistry().contactRetriever(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 6fae732df0a..7afa10ab8d5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -59,7 +59,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new SystemUpgrader(controller, intervals.systemUpgrader)); maintainers.add(new JobRunner(controller, intervals.jobRunner)); maintainers.add(new OsVersionStatusUpdater(controller, intervals.osVersionStatusUpdater)); - maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer)); + maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer, successFactorBaseline.contactInformationMaintainerBaseline)); maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher)); maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer())); maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().resourceDatabase())); @@ -85,6 +85,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new EnclaveAccessMaintainer(controller, intervals.defaultInterval)); maintainers.add(new CertificatePoolMaintainer(controller, metric, intervals.certificatePoolMaintainer)); maintainers.add(new BillingReportMaintainer(controller, intervals.billingReportMaintainer)); + maintainers.add(new CloudAccountVerifier(controller, intervals.cloudAccountVerifier)); } public Upgrader upgrader() { return upgrader; } @@ -147,6 +148,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration meteringMonitorMaintainer; private final Duration certificatePoolMaintainer; private final Duration billingReportMaintainer; + private final Duration cloudAccountVerifier; public Intervals(SystemName system) { this.system = Objects.requireNonNull(system); @@ -184,6 +186,7 @@ public class ControllerMaintenance extends AbstractComponent { this.meteringMonitorMaintainer = duration(30, MINUTES); this.certificatePoolMaintainer = duration(15, MINUTES); this.billingReportMaintainer = duration(60, MINUTES); + this.cloudAccountVerifier = duration(10, MINUTES); } private Duration duration(long amount, TemporalUnit unit) { @@ -201,12 +204,14 @@ public class ControllerMaintenance extends AbstractComponent { private final Double deploymentMetricsMaintainerBaseline; private final Double trafficFractionUpdater; private final Double deploymentInfoMaintainerBaseline; + private final Double contactInformationMaintainerBaseline; public SuccessFactorBaseline(SystemName system) { Objects.requireNonNull(system); this.deploymentMetricsMaintainerBaseline = 0.90; this.trafficFractionUpdater = system.isCd() ? 0.5 : 0.65; this.deploymentInfoMaintainerBaseline = system.isCd() ? 0.5 : 0.95; + this.contactInformationMaintainerBaseline = 0.95; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java index 5218da91c46..6c1c4daa1bb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java @@ -33,7 +33,7 @@ public class EnclaveAccessMaintainer extends ControllerMaintainer { private Set<CloudAccount> externalAccounts() { Set<CloudAccount> accounts = new HashSet<>(); for (Tenant tenant : controller().tenants().asList()) - accounts.addAll(controller().applications().accountsOf(tenant.name())); + tenant.cloudAccounts().forEach(accountInfo -> accounts.add(accountInfo.cloudAccount())); return accounts; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index c90fcb81c71..805bf3d7ada 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -67,7 +67,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { private final EndpointSecretManager endpointSecretManager; private final EndpointCertificateProvider endpointCertificateProvider; final Comparator<EligibleJob> oldestFirst = Comparator.comparing(e -> e.deployment.at()); - final BooleanFlag assignRandomizedId; private final StringFlag endpointCertificateAlgo; private final BooleanFlag useAlternateCertProvider; private final IntFlag assignRandomizedIdRate; @@ -81,7 +80,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { this.endpointSecretManager = controller.serviceRegistry().secretManager(); this.curator = controller().curator(); this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); - this.assignRandomizedId = Flags.ASSIGN_RANDOMIZED_ID.bindTo(controller.flagSource()); this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); this.assignRandomizedIdRate = Flags.ASSIGNED_RANDOMIZED_ID_RATE.bindTo(controller.flagSource()); @@ -283,7 +281,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { assignedCertificates.stream() .filter(c -> c.instance().isPresent()) .filter(c -> c.certificate().randomizedId().isEmpty()) - .filter(c -> assignRandomizedId.with(FetchVector.Dimension.INSTANCE_ID, c.application().instance(c.instance().get()).serializedForm()).value()) .filter(c -> controller().applications().getApplication(c.application()).isPresent()) // In case application has been deleted, but certificate is pending deletion .limit(assignRandomizedIdRate.value()) .forEach(c -> assignRandomizedId(c.application(), c.instance().get())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index a25aa9797ba..dc9c4650191 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -7,6 +7,7 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.ClusterSpec.Id; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; @@ -602,7 +603,7 @@ public class CuratorDb { public List<DnsChallenge> readDnsChallenges(DeploymentId id) { return curator.getChildren(dnsChallengePath(id)).stream() - .map(cluster -> readDnsChallenge(new ClusterId(id, ClusterSpec.Id.from(cluster)))) + .map(cluster -> readDnsChallenge(new ClusterId(id, Id.from(cluster)))) .toList(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index e3d61c81667..760fb9b0366 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.security.KeyUtils; import com.yahoo.slime.ArrayTraverser; @@ -20,6 +22,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.Email; @@ -85,6 +88,9 @@ public class TenantSerializer { private static final String invalidateUserSessionsBeforeField = "invalidateUserSessionsBefore"; private static final String tenantRolesLastMaintainedField = "tenantRolesLastMaintained"; private static final String billingReferenceField = "billingReference"; + private static final String cloudAccountsField = "cloudAccounts"; + private static final String accountField = "account"; + private static final String templateVersionField = "templateVersion"; private static final String awsIdField = "awsId"; private static final String roleField = "role"; @@ -97,6 +103,7 @@ public class TenantSerializer { tenantObject.setLong(createdAtField, tenant.createdAt().toEpochMilli()); toSlime(tenant.lastLoginInfo(), tenantObject.setObject(lastLoginInfoField)); tenantObject.setLong(tenantRolesLastMaintainedField, tenant.tenantRolesLastMaintained().toEpochMilli()); + cloudAccountsToSlime(tenant.cloudAccounts(), tenantObject.setArray(cloudAccountsField)); switch (tenant.type()) { case athenz: toSlime((AthenzTenant) tenant, tenantObject); break; @@ -162,6 +169,14 @@ public class TenantSerializer { } } + private void cloudAccountsToSlime(List<CloudAccountInfo> cloudAccounts, Cursor cloudAccountsObject) { + cloudAccounts.forEach(cloudAccountInfo -> { + Cursor object = cloudAccountsObject.addObject(); + object.setString(accountField, cloudAccountInfo.cloudAccount().account()); + object.setString(templateVersionField, cloudAccountInfo.templateVersion().toFullString()); + }); + } + public Tenant tenantFrom(Slime slime) { Inspector tenantObject = slime.get(); Tenant.Type type = typeOf(tenantObject.field(typeField).asString()); @@ -183,7 +198,8 @@ public class TenantSerializer { Instant createdAt = SlimeUtils.instant(tenantObject.field(createdAtField)); LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField)); Instant tenantRolesLastMaintained = SlimeUtils.instant(tenantObject.field(tenantRolesLastMaintainedField)); - return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained); + List<CloudAccountInfo> cloudAccountInfos = cloudAccountsFromSlime(tenantObject.field(cloudAccountsField)); + return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccountInfos); } private CloudTenant cloudTenantFrom(Inspector tenantObject) { @@ -197,8 +213,9 @@ public class TenantSerializer { ArchiveAccess archiveAccess = archiveAccessFromSlime(tenantObject); Optional<Instant> invalidateUserSessionsBefore = SlimeUtils.optionalInstant(tenantObject.field(invalidateUserSessionsBeforeField)); Instant tenantRolesLastMaintained = SlimeUtils.instant(tenantObject.field(tenantRolesLastMaintainedField)); + List<CloudAccountInfo> cloudAccountInfos = cloudAccountsFromSlime(tenantObject.field(cloudAccountsField)); Optional<BillingReference> billingReference = billingReferenceFrom(tenantObject.field(billingReferenceField)); - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, billingReference); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccountInfos, billingReference); } private DeletedTenant deletedTenantFrom(Inspector tenantObject) { @@ -284,6 +301,14 @@ public class TenantSerializer { return new LastLoginInfo(lastLoginByUserLevel); } + private List<CloudAccountInfo> cloudAccountsFromSlime(Inspector cloudAccountsObject) { + return SlimeUtils.entriesStream(cloudAccountsObject) + .map(inspector -> new CloudAccountInfo( + CloudAccount.from(inspector.field(accountField).asString()), + Version.fromString(inspector.field(templateVersionField).asString()))) + .toList(); + } + void toSlime(TenantInfo info, Cursor parentCursor) { if (info.isEmpty()) return; Cursor infoCursor = parentCursor.setObject("info"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 46c81fc073f..16d862a66ef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2915,6 +2915,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } } tenantMetaDataToSlime(tenant, applications, object.setObject("metaData")); + + if (!tenant.cloudAccounts().isEmpty()) { + Cursor cloudAccounts = object.setArray("cloudAccounts"); + tenant.cloudAccounts().forEach(accountInfo -> { + Cursor accountObject = cloudAccounts.addObject(); + accountObject.setString("cloudAccount", accountInfo.cloudAccount().value()); + accountObject.setString("templateVersion", accountInfo.templateVersion().toFullString()); + }); + } } private void toSlime(ArchiveAccess archiveAccess, Cursor object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index 67dd172fd83..c5fb1afbae8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -85,6 +85,8 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler .addRoute(RestApi.route("/billing/v2/accountant/preview/tenant/{tenant}") .get(self::previewBill) .post(Slime.class, self::createBill)) + .addRoute(RestApi.route("/billing/v2/accountant/bill/{invoice}/export") + .put(Slime.class, self::putAccountantInvoiceExport)) .addRoute(RestApi.route("/billing/v2/accountant/plans") .get(self::plans)) .addExceptionMapper(RuntimeException.class, (c, e) -> ErrorResponses.logThrowing(c.request(), log, e)) @@ -262,6 +264,19 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return new SlimeJsonResponse(slime); } + private HttpResponse putAccountantInvoiceExport(RestApi.RequestContext ctx, Slime slime) { + var billId = ctx.attributes().get("invoice") + .map(id -> Bill.Id.of((String) id)) + .orElseThrow(() -> new RestApiException.BadRequest("Missing bill ID")); + + // TODO: try to find a way to retrieve the cloud tenant from BillingControllerImpl + var bill = billing.getBill(billId); + var cloudTenant = tenants.require(bill.tenant(), CloudTenant.class); + + var exportMethod = slime.get().field("method").asString(); + var result = billing.exportBill(bill, exportMethod, cloudTenant); + return new MessageResponse("Bill has been exported: " + result); + } // --------- INVOICE RENDERING ---------- diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index de25161c461..a21c6548a0b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -5,6 +5,7 @@ import ai.vespa.http.DomainName; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.AuthMethod; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; @@ -263,8 +264,14 @@ public class RoutingPolicies { } else { weightedEndpoints = weightedEndpoints.not().generated(); } + if (generated && weightedEndpoints.isEmpty()) { + // Ignore this policy. If an instance has a global endpoint, and is switching from non-generated to + // generated endpoints we cannot update global DNS record for a deployment until it has been deployed at + // least once (which assigns a generated endpoint). + continue; + } if (weightedEndpoints.size() != 1) { - throw new IllegalStateException("Expected to compute exactly one region endpoint for " + policy.id() + " with parent " + parent); + throw new IllegalStateException("Expected to compute exactly one region endpoint for " + policy.id() + " with parent " + parent + ", got " + weightedEndpoints); } Endpoint endpoint = weightedEndpoints.first().get(); RegionEndpoint regionEndpoint = endpoints.computeIfAbsent(endpoint, (k) -> new RegionEndpoint( @@ -410,24 +417,22 @@ public class RoutingPolicies { new Record(Record.Type.CNAME, name, RecordData.fqdn(policy.canonicalName().get().value())) : new Record(Record.Type.A, name, RecordData.from(policy.ipAddress().orElseThrow())); nameServiceForwarder(endpoint).createRecord(record, Priority.normal, ownerOf(deploymentId)); - setPrivateDns(endpoint, loadBalancer, deploymentId); } + setPrivateDns(zoneEndpoints, loadBalancer, deploymentId); } - private void setPrivateDns(Endpoint endpoint, LoadBalancer loadBalancer, DeploymentId deploymentId) { + private void setPrivateDns(EndpointList endpoints, LoadBalancer loadBalancer, DeploymentId deploymentId) { if (loadBalancer.service().isEmpty()) return; - // TODO(mpolden): Why is this done? Consider creating private DNS for all auth methods - boolean skipBasedOnAuthMethod = switch (endpoint.authMethod()) { - case token -> true; - case mtls -> false; - case none -> true; - }; - if (skipBasedOnAuthMethod) return; + // TODO(mpolden): Model one service for each endpoint (type), to allow private endpoints with tokens. + EndpointList mtlsEndpoints = endpoints.authMethod(AuthMethod.mtls); + if (mtlsEndpoints.isEmpty()) return; + Endpoint endpoint = mtlsEndpoints.generated().first().orElse(mtlsEndpoints.first().get()); if (endpoint.routingMethod() != RoutingMethod.exclusive) return; // Not supported for this routing method controller.serviceRegistry().vpcEndpointService() .setPrivateDns(DomainName.of(endpoint.dnsName()), new ClusterId(deploymentId, endpoint.cluster()), - loadBalancer.cloudAccount()) + loadBalancer.cloudAccount(), + endpoint.generated().isPresent()) .ifPresent(challenge -> { try (Mutex lock = db.lockNameServiceQueue()) { controller.nameServiceForwarder().createTxt(challenge.name(), List.of(challenge.data()), Priority.high, ownerOf(deploymentId)); @@ -436,10 +441,18 @@ public class RoutingPolicies { }); } + /** Deletes all DNS challenges, and corresponding TXT records, for the given deployment. */ + public void removeDnsChallenges(DeploymentId deploymentId) { + try (Mutex lock = db.lockNameServiceQueue()) { + db.readDnsChallenges(deploymentId).forEach(this::removeDnsChallenge); + } + } + /** Returns true iff. the given deployment has no incomplete DNS challenges, or throws (and cleans up) on errors. */ public boolean processDnsChallenges(DeploymentId deploymentId) { try (Mutex lock = db.lockNameServiceQueue()) { List<DnsChallenge> challenges = new ArrayList<>(db.readDnsChallenges(deploymentId)); + challenges.removeIf(challenge -> challenge.state() == ChallengeState.done); Set<RecordName> pendingRequests = controller.curator().readNameServiceQueue().requests().stream() .map(NameServiceRequest::name) .collect(Collectors.toSet()); @@ -450,14 +463,8 @@ public class RoutingPolicies { challenge = challenge.withState(ChallengeState.ready); } ChallengeState state = controller.serviceRegistry().vpcEndpointService().process(challenge); - if (state == ChallengeState.done) { - removeDnsChallenge(challenge); - return true; - } - else { - db.writeDnsChallenge(challenge.withState(state)); - return false; - } + db.writeDnsChallenge(challenge.withState(state)); + return state == ChallengeState.done; }); return challenges.isEmpty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java index df0226176a2..99f60735f6e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java @@ -57,6 +57,7 @@ public abstract class DeploymentRoutingContext implements RoutingContext { /** Deactivate routing configuration for the deployment in this context, using given deployment spec */ public final void deactivate(DeploymentSpec deploymentSpec) { routing.policies().refresh(deployment, deploymentSpec, EndpointList.EMPTY); + routing.policies().removeDnsChallenges(deployment); } /** Routing method of this context */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index 1cb43453918..a6d3b435dcb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -306,6 +306,9 @@ public class EndpointCertificatesTest { fail("Expected exception as certificate is not ready"); } catch (IllegalArgumentException ignored) {} + // Advance clock to verify last requested time + clock.advance(Duration.ofDays(3)); + // Certificate is assigned from pool instead. The previously assigned certificate will eventually be cleaned up // by EndpointCertificateMaintainer { // prod @@ -315,6 +318,7 @@ public class EndpointCertificatesTest { assertEquals(certId, cert.get().randomizedId().get()); assertEquals(certId, tester.curator().readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.empty()).get().certificate().randomizedId().get(), "Certificate is assigned at application-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); + assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); } { // dev @@ -325,6 +329,7 @@ public class EndpointCertificatesTest { assertEquals(certId, cert.get().randomizedId().get()); assertEquals(certId, tester.curator().readAssignedCertificate(instance.id()).get().certificate().randomizedId().get(), "Certificate is assigned at instance-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); + assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index ed5226ebc8b..0e5308fcef5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -42,6 +42,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter import com.yahoo.vespa.hosted.controller.api.integration.configserver.ProxyResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.QuotaUsage; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; +import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.FingerPrint; +import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; @@ -103,6 +105,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Map<DeploymentId, TestReport> testReport = new HashMap<>(); private final Map<DeploymentId, CloudAccount> cloudAccounts = new HashMap<>(); private final Map<DeploymentId, List<X509Certificate>> additionalCertificates = new HashMap<>(); + private final Map<HostName, Map<TokenId, List<FingerPrint>>> activeTokenFingerprints = new HashMap<>(); private List<SearchNodeMetrics> searchNodeMetrics; private Version lastPrepareVersion = null; @@ -319,6 +322,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return additionalCertificates.getOrDefault(deployment, List.of()); } + public void setActiveTokenFingerprints(HostName hostname, Map<TokenId, List<FingerPrint>> tokens) { + activeTokenFingerprints.put(hostname, tokens); + } + @Override public NodeRepositoryMock nodeRepository() { return nodeRepository; @@ -585,6 +592,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return "{\"settings\":{\"name\":\"foo\",\"role\":\"vespa-secretstore-access\",\"awsId\":\"892075328880\",\"externalId\":\"*****\",\"region\":\"us-east-1\"},\"status\":\"ok\"}"; } + @Override + public Map<HostName, Map<TokenId, List<FingerPrint>>> activeTokenFingerprints(DeploymentId deploymentId) { + return activeTokenFingerprints; + } + public static class Application { private final ApplicationId id; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index f2c827478c0..c6386509585 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingControll import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClientMock; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporter; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporterMock; import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; @@ -52,9 +53,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainerMock; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.MockChangeRequestClient; +import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import java.time.Instant; import java.util.Optional; +import java.util.UUID; /** * A mock implementation of a {@link ServiceRegistry} for testing purposes. @@ -316,6 +320,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg @Override public BillingReporter billingReporter() { - return () -> 0.0; + return new BillingReporterMock(clock()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java new file mode 100644 index 00000000000..b1e00ba0746 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; +import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class BillingReportMaintainerTest { + private final ControllerTester tester = new ControllerTester(SystemName.PublicCd); + private final BillingReportMaintainer maintainer = new BillingReportMaintainer(tester.controller(), Duration.ofMinutes(10)); + + @Test + void only_billable_tenants_are_maintained() { + var t1 = tester.createTenant("t1"); + var t2 = tester.createTenant("t2"); + + tester.controller().serviceRegistry().billingController().setPlan(t1, PlanRegistryMock.paidPlan.id(), false, true); + maintainer.maintain(); + + var b1 = billingReference(t1); + var b2 = billingReference(t2); + + assertFalse(b1.isEmpty()); + assertTrue(b2.isEmpty()); + + assertEquals(tester.clock().instant(), b1.orElseThrow().updated()); + assertNotNull(b1.orElseThrow().reference()); + } + + private Optional<BillingReference> billingReference(TenantName tenantName) { + var t = tester.controller().tenants().require(tenantName, CloudTenant.class); + return t.billingReference(); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java index 88c5ae9ff06..4257261b09b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java @@ -53,12 +53,4 @@ public class CertificatePoolMaintainerTest { assertEquals(0.0, maintainer.maintain(), 0.0000001); assertEquals(n, tester.curator().readUnassignedCertificates().size()); } - - void old_unassigned_certs_are_refreshed() { - tester.flagSource().withIntFlag(PermanentFlags.CERT_POOL_SIZE.id(), 1); - assertNumCerts(1); - EndpointCertificateProviderMock endpointCertificateProvider = (EndpointCertificateProviderMock) tester.controller().serviceRegistry().endpointCertificateProvider(); - var request = endpointCertificateProvider.listCertificates().get(0); - - } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java index f0c11c0ddbd..2c54c0c9fb6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java @@ -30,7 +30,7 @@ public class ContactInformationMaintainerTest { @BeforeEach public void before() { tester = new ControllerTester(); - maintainer = new ContactInformationMaintainer(tester.controller(), Duration.ofDays(1)); + maintainer = new ContactInformationMaintainer(tester.controller(), Duration.ofDays(1), 1.0); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainerTest.java index 5bfac2866ce..1e1079a3314 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainerTest.java @@ -21,17 +21,20 @@ class EnclaveAccessMaintainerTest { void test() { ControllerTester tester = new ControllerTester(); MockEnclaveAccessService amis = tester.serviceRegistry().enclaveAccessService(); - EnclaveAccessMaintainer sharer = new EnclaveAccessMaintainer(tester.controller(), Duration.ofMinutes(1)); + EnclaveAccessMaintainer sharer = new EnclaveAccessMaintainer(tester.controller(), Duration.ofHours(1)); + CloudAccountVerifier accountVerifier = new CloudAccountVerifier(tester.controller(), Duration.ofHours(1)); assertEquals(Set.of(), amis.currentAccounts()); assertEquals(1, sharer.maintain()); assertEquals(Set.of(), amis.currentAccounts()); tester.createTenant("tanten"); + accountVerifier.maintain(); assertEquals(1, sharer.maintain()); assertEquals(Set.of(), amis.currentAccounts()); tester.flagSource().withListFlag(PermanentFlags.CLOUD_ACCOUNTS.id(), List.of("123123123123", "321321321321"), String.class); + accountVerifier.maintain(); assertEquals(1, sharer.maintain()); assertEquals(Set.of(CloudAccount.from("aws:123123123123"), CloudAccount.from("aws:321321321321")), amis.currentAccounts()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 918a4bed6f4..cbc69e52119 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -11,7 +11,9 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateDetails; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProviderMock; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateRequest; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -24,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import java.time.Duration; @@ -36,10 +39,13 @@ import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devUsEast1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.perfUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsCentral1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -100,12 +106,15 @@ public class EndpointCertificateMaintainerTest { assertEquals(0.0, maintainer.maintain(), 0.0000001); var cert = tester.curator().readAssignedCertificate(appId).orElseThrow().certificate(); - tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(cert.rootRequestId()); // cert should not be deleted, the app is deployed! + tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(cert.leafRequestId().get()); // cert should not be deleted, the app is deployed! } @Test void refreshed_certificate_is_discovered_and_after_four_days_deployed() { - var appId = ApplicationId.from("tenant", "application", "default"); + prepareCertificatePool(1); + + var instanceId = ApplicationId.from("tenant", "application", "default"); + var applicationId = TenantAndApplicationId.from(instanceId); DeploymentTester deploymentTester = new DeploymentTester(tester); @@ -115,22 +124,25 @@ public class EndpointCertificateMaintainerTest { DeploymentContext deploymentContext = deploymentTester.newDeploymentContext("tenant", "application", "default"); deploymentContext.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1); - var assignedCertificate = tester.curator().readAssignedCertificate(appId).orElseThrow(); + var assignedCertificate = tester.curator().readAssignedCertificate(applicationId, Optional.empty()).orElseThrow(); // cert should not be deleted, the app is deployed! assertEquals(0.0, maintainer.maintain(), 0.0000001); - assertEquals(tester.curator().readAssignedCertificate(appId), Optional.of(assignedCertificate)); + assertEquals(tester.curator().readAssignedCertificate(applicationId, Optional.empty()).map(c->c.certificate().rootRequestId()), Optional.of(assignedCertificate.certificate().rootRequestId())); tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(assignedCertificate.certificate().rootRequestId()); + // TODO: Remove this line when we have removed assignment of randomized id to application certificates + //assignedCertificate = tester.curator().readAssignedCertificate().orElseThrow(); // This simulates a cert refresh performed 3 days later tester.clock().advance(Duration.ofDays(3)); secretStore.setSecret(assignedCertificate.certificate().keyName(), "foo", 1); secretStore.setSecret(assignedCertificate.certificate().certName(), "bar", 1); - tester.controller().serviceRegistry().endpointCertificateProvider().requestCaSignedCertificate(appId.toFullString(), assignedCertificate.certificate().requestedDnsSans(), Optional.of(assignedCertificate.certificate()), "rsa_2048", false); + tester.controller().serviceRegistry().endpointCertificateProvider().requestCaSignedCertificate("preprovisioned." + assignedCertificate.certificate().randomizedId().get(), assignedCertificate.certificate().requestedDnsSans(), Optional.of(assignedCertificate.certificate()), "rsa_2048", false); + // We should now pick up the new key and cert version + uuid, but not force trigger deployment yet assertEquals(0.0, maintainer.maintain(), 0.0000001); deploymentContext.assertNotRunning(productionUsWest1); - var updatedCert = tester.curator().readAssignedCertificate(appId).orElseThrow().certificate(); + var updatedCert = tester.curator().readAssignedCertificate(applicationId, Optional.empty()).orElseThrow().certificate(); assertNotEquals(assignedCertificate.certificate().leafRequestId().orElseThrow(), updatedCert.leafRequestId().orElseThrow()); assertEquals(updatedCert.version(), assignedCertificate.certificate().version() + 1); @@ -179,24 +191,12 @@ public class EndpointCertificateMaintainerTest { } @Test - void certificates_are_not_assigned_random_id_when_flag_disabled() { - var app = ApplicationId.from("tenant", "app", "default"); - DeploymentTester deploymentTester = new DeploymentTester(tester); - deployToAssignCert(deploymentTester, app, List.of(systemTest, stagingTest, productionUsWest1), Optional.empty()); - assertEquals(1, tester.curator().readAssignedCertificates().size()); - - maintainer.maintain(); - assertEquals(1, tester.curator().readAssignedCertificates().size()); - } - - @Test void production_deployment_certificates_are_assigned_random_id() { var app = ApplicationId.from("tenant", "app", "default"); DeploymentTester deploymentTester = new DeploymentTester(tester); deployToAssignCert(deploymentTester, app, List.of(systemTest, stagingTest, productionUsWest1), Optional.empty()); assertEquals(1, tester.curator().readAssignedCertificates().size()); - ((InMemoryFlagSource)deploymentTester.controller().flagSource()).withBooleanFlag(Flags.ASSIGN_RANDOMIZED_ID.id(), true); maintainer.maintain(); assertEquals(2, tester.curator().readAssignedCertificates().size()); @@ -223,7 +223,6 @@ public class EndpointCertificateMaintainerTest { DeploymentTester deploymentTester = new DeploymentTester(tester); deployToAssignCert(deploymentTester, instance1, List.of(systemTest, stagingTest,productionUsWest1),Optional.of("instance1")); assertEquals(1, tester.curator().readAssignedCertificates().size()); - ((InMemoryFlagSource)deploymentTester.controller().flagSource()).withBooleanFlag(Flags.ASSIGN_RANDOMIZED_ID.id(), true); maintainer.maintain(); String randomId = tester.curator().readAssignedCertificate(instance1).get().certificate().randomizedId().get(); @@ -241,7 +240,6 @@ public class EndpointCertificateMaintainerTest { DeploymentTester deploymentTester = new DeploymentTester(tester); deployToAssignCert(deploymentTester, devApp, List.of(devUsEast1), Optional.empty()); assertEquals(1, tester.curator().readAssignedCertificates().size()); - ((InMemoryFlagSource)deploymentTester.controller().flagSource()).withBooleanFlag(Flags.ASSIGN_RANDOMIZED_ID.id(), true); List<String> originalRequestedSans = tester.curator().readAssignedCertificate(devApp).get().certificate().requestedDnsSans(); maintainer.maintain(); assertEquals(1, tester.curator().readAssignedCertificates().size()); @@ -254,9 +252,64 @@ public class EndpointCertificateMaintainerTest { assertEquals(3, randomizedNames.size()); } + @Test + void deploy_to_other_manual_zone_refreshes_cert() { + String devSan = "*.foo.manual.tenant.us-east-1.dev.vespa.oath.cloud"; + String perfSan = "*.foo.manual.tenant.us-east-3.perf.vespa.oath.cloud"; + + var devApp = ApplicationId.from("tenant", "manual", "foo"); + DeploymentTester deploymentTester = new DeploymentTester(tester); + deployToAssignCert(deploymentTester, devApp, List.of(devUsEast1), Optional.empty()); + assertEquals(1, tester.curator().readAssignedCertificates().size()); + maintainer.maintain(); + Optional<AssignedCertificate> devCertificate = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(devApp), Optional.of(devApp.instance())); + List<String> devSans = devCertificate.get().certificate().requestedDnsSans(); + Assertions.assertThat(devSans).contains(devSan); + Assertions.assertThat(devSans).doesNotContain(perfSan); + + // Deploy to perf and verify that the certs are refreshed + deployToAssignCert(deploymentTester, devApp, List.of(perfUsEast3), Optional.empty()); + Optional<AssignedCertificate> devAndPerfCertificate = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(devApp), Optional.of(devApp.instance())); + List<String> devAndPerfSans = devAndPerfCertificate.get().certificate().requestedDnsSans(); + + assertNotEquals(devSans, devAndPerfSans); + Assertions.assertThat(devAndPerfSans).contains(devSan); + Assertions.assertThat(devAndPerfSans).contains(perfSan); + } + + @Test + void deploy_to_other_prod_zone_refreshes_cert() { + String westSan = "*.prod.tenant.us-west-1.vespa.oath.cloud"; + String centralSan = "*.prod.tenant.us-central-1.vespa.oath.cloud"; + + var prodApp = ApplicationId.from("tenant", "prod", "default"); + DeploymentTester deploymentTester = new DeploymentTester(tester); + deployToAssignCert(deploymentTester, prodApp, List.of(systemTest, stagingTest, productionUsWest1), Optional.empty()); + assertEquals(1, tester.curator().readAssignedCertificates().size()); + maintainer.maintain(); + Optional<AssignedCertificate> usWestCert = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(prodApp), Optional.of(prodApp.instance())); + List<String> usWestSans = usWestCert.get().certificate().requestedDnsSans(); + Assertions.assertThat(usWestSans).contains(westSan); + Assertions.assertThat(usWestSans).doesNotContain(centralSan); + + // Deploy to perf and verify that the certs are refreshed + deployToAssignCert(deploymentTester, prodApp, List.of(systemTest, stagingTest, productionUsWest1, productionUsCentral1), Optional.empty()); + Optional<AssignedCertificate> usCentralWestCert = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(prodApp), Optional.of(prodApp.instance())); + List<String> usCentralWestSans = usCentralWestCert.get().certificate().requestedDnsSans(); + assertNotEquals(usWestSans, usCentralWestSans); + Assertions.assertThat(usCentralWestSans).contains(westSan); + Assertions.assertThat(usCentralWestSans).contains(centralSan); + } + + private void deploy() { + + } + private void deployToAssignCert(DeploymentTester tester, ApplicationId applicationId, List<JobType> jobTypes, Optional<String> instances) { - var applicationPackageBuilder = new ApplicationPackageBuilder() - .region("us-west-1"); + + var applicationPackageBuilder = new ApplicationPackageBuilder(); + jobTypes.stream().filter(JobType::isProduction).map(job -> job.zone().region().value()).forEach(applicationPackageBuilder::region); + instances.map(applicationPackageBuilder::instances); var applicationPackage = applicationPackageBuilder.build(); @@ -279,4 +332,23 @@ public class EndpointCertificateMaintainerTest { return new AssignedCertificate(TenantAndApplicationId.from(instance), Optional.of(instance.instance()), certificate); } + private void prepareCertificatePool(int numCertificates) { + ((InMemoryFlagSource)tester.controller().flagSource()).withIntFlag(PermanentFlags.CERT_POOL_SIZE.id(), numCertificates); + ((InMemoryFlagSource)tester.controller().flagSource()).withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + + // Provision certificates + for (int i = 0; i < numCertificates; i++) { + certificatePoolMaintainer.maintain(); + } + + // Make certificate ready + EndpointCertificateProviderMock endpointCertificateProvider = (EndpointCertificateProviderMock) tester.controller().serviceRegistry().endpointCertificateProvider(); + List<EndpointCertificateRequest> endpointCertificateRequests = endpointCertificateProvider.listCertificates(); + endpointCertificateRequests.forEach(cert -> { + EndpointCertificateDetails details = endpointCertificateProvider.certificateDetails(cert.requestId()); + secretStore.setSecret(details.privateKeyKeyname(), "foo", 0); + secretStore.setSecret(details.certKeyKeyname(), "bar", 0); + }); + certificatePoolMaintainer.maintain(); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index bdbbc4b293f..228a61cebc6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; import com.yahoo.test.ManualClock; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; @@ -69,6 +68,7 @@ public class NotificationsDbTest { new ArchiveAccess(), Optional.empty(), Instant.EPOCH, + List.of(), Optional.empty()); private static final List<Notification> notifications = List.of( notification(1001, Type.deployment, Level.error, NotificationSource.from(tenant), "tenant msg"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java index ef1d9cd92e3..15524e2748c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; @@ -47,6 +46,7 @@ public class NotifierTest { new ArchiveAccess(), Optional.empty(), Instant.EPOCH, + List.of(), Optional.empty()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index dd7afa314ea..4369675ba3e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.controller.persistence;// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import com.google.common.collect.ImmutableBiMap; +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.security.KeyUtils; import com.yahoo.slime.Cursor; @@ -16,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.BillingReference; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.Email; @@ -91,7 +94,8 @@ public class TenantSerializerTest { Optional.of(contact()), Instant.EPOCH, lastLoginInfo(321L, 654L, 987L), - Instant.EPOCH); + Instant.EPOCH, + List.of()); AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.contact(), serialized.contact()); } @@ -109,6 +113,7 @@ public class TenantSerializerTest { new ArchiveAccess(), Optional.empty(), Instant.EPOCH, + List.of(), Optional.empty()); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.name(), serialized.name()); @@ -133,6 +138,7 @@ public class TenantSerializerTest { new ArchiveAccess().withAWSRole("arn:aws:iam::123456789012:role/my-role"), Optional.of(Instant.ofEpochMilli(1234567)), Instant.EPOCH, + List.of(), Optional.empty()); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.info(), serialized.info()); @@ -185,6 +191,8 @@ public class TenantSerializerTest { new ArchiveAccess().withAWSRole("arn:aws:iam::123456789012:role/my-role").withGCPMember("user:foo@example.com"), Optional.empty(), Instant.EPOCH, + List.of(new CloudAccountInfo(CloudAccount.from("aws:123456789012"), Version.fromString("1.2.3")), + new CloudAccountInfo(CloudAccount.from("gcp:my-project"), Version.fromString("3.2.1"))), Optional.empty()); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(serialized.archiveAccess().awsRole().get(), "arn:aws:iam::123456789012:role/my-role"); @@ -263,7 +271,8 @@ public class TenantSerializerTest { Optional.of(contact()), Instant.EPOCH, lastLoginInfo(321L, 654L, 987L), - Instant.ofEpochMilli(1_000_000)); + Instant.ofEpochMilli(1_000_000), + List.of()); assertEquals(tenant, serializer.tenantFrom(serializer.toSlime(tenant))); } @@ -281,6 +290,7 @@ public class TenantSerializerTest { new ArchiveAccess().withAWSRole("arn:aws:iam::123456789012:role/my-role").withGCPMember("user:foo@example.com"), Optional.empty(), Instant.EPOCH, + List.of(), Optional.of(reference)); var slime = serializer.toSlime(tenant); var deserialized = serializer.tenantFrom(slime); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 4eb6e080737..3b74fea2b9c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -5,6 +5,7 @@ import ai.vespa.hosted.api.MultiPartStreamer; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.RestApiException; @@ -26,12 +27,14 @@ import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; import com.yahoo.vespa.hosted.controller.security.Auth0Credentials; import com.yahoo.vespa.hosted.controller.security.CloudTenantSpec; import com.yahoo.vespa.hosted.controller.security.Credentials; +import com.yahoo.vespa.hosted.controller.tenant.CloudAccountInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.File; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -369,10 +372,10 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { new DeploymentTester(wrapped).newDeploymentContext(ApplicationId.from(tenantName, applicationName, InstanceName.defaultName())) .submit() .deploy(); + tester.controller().tenants().updateCloudAccounts(tenantName, List.of(new CloudAccountInfo(CloudAccount.from("aws:123456789012"), new Version(1, 2, 4)))); tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), - (response) -> assertFalse(response.getBodyAsString().contains("archiveAccessRole")), - 200); + new File("tenant-cloud.json")); tester.assertResponse(request("/application/v4/tenant/scoober/archive-access/aws", PUT) .data("{\"role\":\"arn:aws:iam::123456789012:role/my-role\"}").roles(Role.administrator(tenantName)), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index ab70dfd6073..6b377e2069b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -1372,7 +1372,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Create legacy tenant name containing underscores tester.controller().curator().writeTenant(new AthenzTenant(TenantName.from("my_tenant"), ATHENZ_TENANT_DOMAIN, - new Property("property1"), Optional.empty(), Optional.empty(), Instant.EPOCH, LastLoginInfo.EMPTY, Instant.EPOCH)); + new Property("property1"), Optional.empty(), Optional.empty(), Instant.EPOCH, LastLoginInfo.EMPTY, Instant.EPOCH, List.of())); // POST (add) a Athenz tenant with dashes duplicates existing one with underscores tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-cloud.json new file mode 100644 index 00000000000..c7258ab3aa6 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-cloud.json @@ -0,0 +1,35 @@ +{ + "tenant": "scoober", + "type": "CLOUD", + "creator": "developer@scoober", + "pemDeveloperKeys": [], + "secretStores": [], + "integrations": { + "aws": { + "tenantRole": "scoober-tenant-role", + "accounts": [] + } + }, + "quota": { + "budgetUsed": 1.304 + }, + "archiveAccess": {}, + "applications": [ + { + "tenant": "scoober", + "application": "albums", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/scoober/application/albums/instance/default" + } + ], + "metaData": { + "createdAtMillis": 1600000000000, + "lastSubmissionToProdMillis": 1000 + }, + "cloudAccounts": [ + { + "cloudAccount": "aws:123456789012", + "templateVersion": "1.2.4" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index eb376a95c74..8b76613676c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -34,6 +34,9 @@ "name": "ChangeRequestMaintainer" }, { + "name": "CloudAccountVerifier" + }, + { "name": "CloudDatabaseMaintainer" }, { @@ -130,7 +133,5 @@ "name": "VersionStatusUpdater" } ], - "inactive": [ - "DeploymentExpirer" - ] + "inactive": ["DeploymentExpirer"] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index 581f9704fc5..001e02e1b16 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -70,17 +70,7 @@ public class SignatureFilterTest { filter = new SignatureFilter(tester.controller()); signer = new RequestSigner(privateKey, id.serializedForm(), tester.clock()); - tester.curator().writeTenant(new CloudTenant(appId.tenant(), - Instant.EPOCH, - LastLoginInfo.EMPTY, - Optional.empty(), - ImmutableBiMap.of(), - TenantInfo.empty(), - List.of(), - new ArchiveAccess(), - Optional.empty(), - Instant.EPOCH, - Optional.empty())); + tester.curator().writeTenant(CloudTenant.create(appId.tenant(), Instant.EPOCH, null)); tester.curator().writeApplication(new Application(appId, tester.clock().instant())); } @@ -129,6 +119,7 @@ public class SignatureFilterTest { new ArchiveAccess(), Optional.empty(), Instant.EPOCH, + List.of(), Optional.empty())); verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), new SecurityContext(new SimplePrincipal("user"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java index 779aee73dae..eb3f9daef53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java @@ -63,7 +63,7 @@ public class UserFlagsSerializerTest { "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB "{\"id\":\"jackson-id\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"tenant\"}],\"value\":{\"integer\":456,\"string\":\"xyz\"}},{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Resolved for email // Resolved for email, but conditions are empty since this user is not authorized for any tenants - "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\"}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\"}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\"}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\"}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"value1\"}]}]}", // resolved for email flagData, Set.of(), false, email1); @@ -72,7 +72,7 @@ public class UserFlagsSerializerTest { "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB "{\"id\":\"jackson-id\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"tenant\",\"values\":[\"tenant1\"]}],\"value\":{\"integer\":456,\"string\":\"xyz\"}},{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Resolved for email // Resolved for email, but conditions have filtered out tenant2 - "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"value1\"}]}]}", // resolved for email flagData, Set.of("tenant1"), false, email1); @@ -81,7 +81,7 @@ public class UserFlagsSerializerTest { "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB "{\"id\":\"jackson-id\",\"rules\":[{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Default from code, no DB values match // Includes last value from DB which is not conditioned on email and the default from code - "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\",\"tenant2:music:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\",\"tenant2:music:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"default value\"}]}]}", // Default from code flagData, Set.of(), true, "operator@domain.tld"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 630de5137bb..b2b34441219 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -598,21 +598,28 @@ public class RoutingPoliciesTest { app.deploy(); - // TXT records are cleaned up as we go—the last challenge is the last to go here, and we must flush it ourselves. + // TXT records are cleaned up when deployments are deactivated. + // The last challenge is the last to go here, and we must flush it ourselves. assertEquals(List.of("a.t.aws-us-east-33a.vespa.oath.cloud", "challenge--a.t.aws-us-east-33a.vespa.oath.cloud"), tester.recordNames()); app.flushDnsUpdates(); assertEquals(Set.of(new Record(Type.CNAME, RecordName.from("a.t.aws-us-east-33a.vespa.oath.cloud"), - RecordData.from("lb-0--t.a.default--prod.aws-us-east-33a."))), + RecordData.from("lb-0--t.a.default--prod.aws-us-east-33a.")), + new Record(Type.TXT, + RecordName.from("challenge--a.t.aws-us-east-33a.vespa.oath.cloud"), + RecordData.from("system"))), tester.controllerTester().nameService().records()); + tester.controllerTester().controller().applications().deactivate(app.instanceId(), zone3); + app.flushDnsUpdates(); + assertEquals(Set.of(), + tester.controllerTester().nameService().records()); + // Deployment fails because challenge is not answered (immediately). tester.tester.controllerTester().serviceRegistry().vpcEndpointService().outcomes .put(RecordName.from("challenge--a.t.aws-us-east-33a.vespa.oath.cloud"), ChallengeState.running); - - // Deployment fails because challenge is not answered (immediately). assertEquals("Status of run 2 of production-aws-us-east-33a for t.a ==> expected: <succeeded> but was: <unfinished>", assertThrows(AssertionError.class, () -> app.submit(appPackage).deploy()) @@ -1057,40 +1064,47 @@ public class RoutingPoliciesTest { int clustersPerZone = 2; var zone1 = ZoneId.from("prod", "aws-us-east-1c"); var zone2 = ZoneId.from("prod", "aws-eu-west-1a"); + var zone3 = ZoneId.from("prod", "aws-us-east-1a"); // To test global endpoint pointing to two zones in same cloud-native region ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) .region(zone2.region()) + .region(zone3.region()) .container("c0", AuthMethod.mtls) .container("c1", AuthMethod.mtls, AuthMethod.token) .endpoint("foo", "c0") .applicationEndpoint("bar", "c0", Map.of(zone1.region().value(), Map.of(InstanceName.defaultName(), 1))) .build(); - tester.provisionLoadBalancers(clustersPerZone, context.instanceId(), zone1, zone2); + tester.provisionLoadBalancers(clustersPerZone, context.instanceId(), zone1, zone2, zone3); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Deployment creates generated zone names List<String> expectedRecords = List.of( // save me, jebus! - "b36bf591.cafed00d.aws-us-east-1.w.vespa-app.cloud", + "a6414896.cafed00d.aws-eu-west-1.w.vespa-app.cloud", "b36bf591.cafed00d.z.vespa-app.cloud", "bar.app1.tenant1.a.vespa-app.cloud", "bc50b636.cafed00d.z.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1.w.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1.w.vespa-app.cloud", + "c0.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", + "c1.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c33db5ed.cafed00d.z.vespa-app.cloud", + "d467800f.cafed00d.z.vespa-app.cloud", "d71005bf.cafed00d.z.vespa-app.cloud", - "dd0971b4.cafed00d.aws-eu-west-1.w.vespa-app.cloud", "dd0971b4.cafed00d.z.vespa-app.cloud", "eb48ad53.cafed00d.z.vespa-app.cloud", + "ec1e1288.cafed00d.z.vespa-app.cloud", "f2fa41ec.cafed00d.g.vespa-app.cloud", + "f411d177.cafed00d.z.vespa-app.cloud", "f4a4d111.cafed00d.a.vespa-app.cloud", + "fcf1bd63.cafed00d.aws-us-east-1.w.vespa-app.cloud", "foo.app1.tenant1.g.vespa-app.cloud" ); assertEquals(expectedRecords, tester.recordNames()); - assertEquals(4, tester.policiesOf(context.instanceId()).size()); + assertEquals(6, tester.policiesOf(context.instanceId()).size()); ClusterSpec.Id cluster0 = ClusterSpec.Id.from("c0"); ClusterSpec.Id cluster1 = ClusterSpec.Id.from("c1"); for (var zone : List.of(zone1, zone2)) { @@ -1107,13 +1121,17 @@ public class RoutingPoliciesTest { // Ordinary endpoints point to expected targets tester.assertTargets(context.instanceId(), EndpointId.of("foo"), cluster0, 0, - Map.of(zone1, 1L, zone2, 1L)); + ImmutableMap.of(zone1, 1L, + zone2, 1L, + zone3, 1L)); tester.assertTargets(context.application().id(), EndpointId.of("bar"), cluster0, 0, Map.of(context.deploymentIdIn(zone1), 1)); // Generated endpoints point to expected targets tester.assertTargets(context.instanceId(), EndpointId.of("foo"), cluster0, 0, - Map.of(zone1, 1L, zone2, 1L), + ImmutableMap.of(zone1, 1L, + zone2, 1L, + zone3, 1L), true); tester.assertTargets(context.application().id(), EndpointId.of("bar"), cluster0, 0, Map.of(context.deploymentIdIn(zone1), 1), @@ -1127,6 +1145,7 @@ public class RoutingPoliciesTest { // One endpoint is removed applicationPackage = applicationPackageBuilder().region(zone1.region()) .region(zone2.region()) + .region(zone3.region()) .container("c0", AuthMethod.mtls) .container("c1", AuthMethod.mtls, AuthMethod.token) .applicationEndpoint("bar", "c0", Map.of(zone1.region().value(), Map.of(InstanceName.defaultName(), 1))) @@ -1138,13 +1157,18 @@ public class RoutingPoliciesTest { "bar.app1.tenant1.a.vespa-app.cloud", "bc50b636.cafed00d.z.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", + "c0.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", + "c1.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c33db5ed.cafed00d.z.vespa-app.cloud", + "d467800f.cafed00d.z.vespa-app.cloud", "d71005bf.cafed00d.z.vespa-app.cloud", "dd0971b4.cafed00d.z.vespa-app.cloud", "eb48ad53.cafed00d.z.vespa-app.cloud", + "ec1e1288.cafed00d.z.vespa-app.cloud", + "f411d177.cafed00d.z.vespa-app.cloud", "f4a4d111.cafed00d.a.vespa-app.cloud" ), tester.recordNames()); @@ -1157,6 +1181,35 @@ public class RoutingPoliciesTest { } @Test + public void generated_endpoints_only() { + var tester = new RoutingPoliciesTester(SystemName.Public); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + tester.controllerTester().flagSource() + .withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true) + .withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); + addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); + + // Deploy application + var zone1 = ZoneId.from("prod", "aws-us-east-1c"); + ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) + .container("c0", AuthMethod.mtls) + .endpoint("foo", "c0") + .build(); + tester.provisionLoadBalancers(1, context.instanceId(), zone1); + // ConfigServerMock provisions a load balancer for the "default" cluster, but in this scenario we need full + // control over the load balancer name because "default" has no special treatment when using generated endpoints + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); + tester.assertTargets(context.instance().id(), EndpointId.of("foo"), ClusterSpec.Id.from("c0"), + 0, Map.of(zone1, 1L), true); + assertEquals(List.of("a9c8c045.cafed00d.g.vespa-app.cloud", + "ebd395b6.cafed00d.z.vespa-app.cloud", + "fcf1bd63.cafed00d.aws-us-east-1.w.vespa-app.cloud"), + tester.recordNames()); + } + + @Test public void generated_endpoints_multi_instance() { var tester = new RoutingPoliciesTester(SystemName.Public); var context0 = tester.newDeploymentContext("tenant1", "app1", "default"); @@ -1213,6 +1266,32 @@ public class RoutingPoliciesTest { assertEquals(List.of(), tester.recordNames()); } + @Test + public void generated_endpoint_migration_with_global_endpoint() { + var tester = new RoutingPoliciesTester(SystemName.Public); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); + + // Deploy application + int clustersPerZone = 2; + var zone1 = ZoneId.from("prod", "aws-us-east-1c"); + var zone2 = ZoneId.from("prod", "aws-eu-west-1a"); + ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) + .region(zone2.region()) + .container("c0", AuthMethod.mtls) + .endpoint("foo", "c0") + .build(); + tester.provisionLoadBalancers(clustersPerZone, context.instanceId(), zone1, zone2); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.of("foo"), 0, zone1, zone2); + + // Switch to generated + tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context.instance().id(), EndpointId.of("foo"), ClusterSpec.Id.from("c0"), + 0, Map.of(zone1, 1L, zone2, 1L), true); + } + private void addCertificateToPool(String id, UnassignedCertificate.State state, RoutingPoliciesTester tester) { EndpointCertificate cert = new EndpointCertificate("testKey", "testCert", 1, 0, "request-id", @@ -1270,6 +1349,11 @@ public class RoutingPoliciesTest { .withCloudNativeRegionName("eu-west-1") .build(), ZoneApiMock.newBuilder() + .with(ZoneId.from(Environment.prod, RegionName.from("aws-us-east-1a"))) + .with(CloudName.AWS) + .withCloudNativeRegionName("us-east-1") + .build(), + ZoneApiMock.newBuilder() .with(ZoneId.from(Environment.prod, RegionName.from("gcp-us-south1-b"))) .with(CloudName.GCP) .withCloudNativeRegionName("us-south1") diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 962d666bf6b..3127233f67f 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -34,7 +34,7 @@ <!-- DO NOT UPGRADE THESE TO A NEW MAJOR VERSION WITHOUT CHECKING FOR BINARY COMPATIBILITY --> <aopalliance.vespa.version>1.0</aopalliance.vespa.version> <commons-logging.vespa.version>1.2</commons-logging.vespa.version> <!-- This version is exported by jdisc via jcl-over-slf4j. --> - <error-prone-annotations.vespa.version>2.21.1</error-prone-annotations.vespa.version> + <error-prone-annotations.vespa.version>2.22.0</error-prone-annotations.vespa.version> <guava.vespa.version>32.1.2-jre</guava.vespa.version> <guice.vespa.version>6.0.0</guice.vespa.version> <jackson2.vespa.version>2.15.2</jackson2.vespa.version> @@ -86,7 +86,7 @@ <commons.math3.vespa.version>3.6.1</commons.math3.vespa.version> <commons-compress.vespa.version>1.24.0</commons-compress.vespa.version> <curator.vespa.version>5.5.0</curator.vespa.version> - <dropwizard.metrics.vespa.version>4.2.19</dropwizard.metrics.vespa.version> + <dropwizard.metrics.vespa.version>4.2.20</dropwizard.metrics.vespa.version> <eclipse-collections.vespa.version>11.1.0</eclipse-collections.vespa.version> <felix.vespa.version>7.0.5</felix.vespa.version> <felix.log.vespa.version>1.3.0</felix.log.vespa.version> @@ -106,13 +106,13 @@ <junit.platform.vespa.version>1.10.0</junit.platform.vespa.version> <junit4.vespa.version>4.13.2</junit4.vespa.version> <luben.zstd.vespa.version>1.5.5-5</luben.zstd.vespa.version> - <lucene.vespa.version>9.7.0</lucene.vespa.version> + <lucene.vespa.version>9.8.0</lucene.vespa.version> <maven-archiver.vespa.version>3.6.1</maven-archiver.vespa.version> <maven-wagon.vespa.version>3.5.3</maven-wagon.vespa.version> <mimepull.vespa.version>1.10.0</mimepull.vespa.version> <mockito.vespa.version>5.5.0</mockito.vespa.version> <mojo-executor.vespa.version>2.4.0</mojo-executor.vespa.version> - <netty.vespa.version>4.1.98.Final</netty.vespa.version> + <netty.vespa.version>4.1.99.Final</netty.vespa.version> <netty-tcnative.vespa.version>2.0.61.Final</netty-tcnative.vespa.version> <onnxruntime.vespa.version>1.15.1</onnxruntime.vespa.version> <opennlp.vespa.version>2.3.0</opennlp.vespa.version> @@ -123,9 +123,9 @@ <protobuf.vespa.version>3.24.3</protobuf.vespa.version> <questdb.vespa.version>7.3.2</questdb.vespa.version> <spifly.vespa.version>1.3.6</spifly.vespa.version> - <snappy.vespa.version>1.1.10.3</snappy.vespa.version> + <snappy.vespa.version>1.1.10.5</snappy.vespa.version> <surefire.vespa.version>3.1.2</surefire.vespa.version> - <wiremock.vespa.version>3.1.0</wiremock.vespa.version> + <wiremock.vespa.version>3.2.0</wiremock.vespa.version> <xerces.vespa.version>2.12.2</xerces.vespa.version> <zero-allocation-hashing.vespa.version>0.16</zero-allocation-hashing.vespa.version> <zookeeper.client.vespa.version>3.8.0</zookeeper.client.vespa.version> @@ -153,7 +153,7 @@ <maven-plugin-api.vespa.version>${maven-core.vespa.version}</maven-plugin-api.vespa.version> <maven-plugin-tools.vespa.version>3.9.0</maven-plugin-tools.vespa.version> <maven-resources-plugin.vespa.version>3.3.1</maven-resources-plugin.vespa.version> - <maven-shade-plugin.vespa.version>3.5.0</maven-shade-plugin.vespa.version> + <maven-shade-plugin.vespa.version>3.5.1</maven-shade-plugin.vespa.version> <maven-site-plugin.vespa.version>3.12.1</maven-site-plugin.vespa.version> <maven-source-plugin.vespa.version>3.3.0</maven-source-plugin.vespa.version> <properties-maven-plugin.vespa.version>1.2.0</properties-maven-plugin.vespa.version> diff --git a/document/src/vespa/document/select/parse_utils.cpp b/document/src/vespa/document/select/parse_utils.cpp index 4c116d5bff4..d1e559f211d 100644 --- a/document/src/vespa/document/select/parse_utils.cpp +++ b/document/src/vespa/document/select/parse_utils.cpp @@ -24,7 +24,7 @@ parse_i64(const char* str, size_t len, int64_t& out) { } bool parse_double(const char* str, size_t len, double& out) { -#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 170000 +#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 180000 // Temporary workaround that also handles underflow (cf. issue 3081) // until libc++ supports std::from_chars for double char *str_end = const_cast<char*>(str) + len; diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index 3fafb8f8b18..d0db56f3433 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -14,7 +14,9 @@ #include <llvm/ExecutionEngine/ExecutionEngine.h> #include <llvm/IR/DataLayout.h> #include <llvm/Transforms/Scalar.h> +#if LLVM_VERSION_MAJOR < 17 #include <llvm/Transforms/IPO/PassManagerBuilder.h> +#endif #include <llvm/Support/ManagedStatic.h> #include <vespa/eval/eval/check_type.h> #include <vespa/vespalib/stllike/hash_set.h> diff --git a/eval/src/vespa/eval/eval/tensor_function.cpp b/eval/src/vespa/eval/eval/tensor_function.cpp index 97590322a50..7bef6d2d880 100644 --- a/eval/src/vespa/eval/eval/tensor_function.cpp +++ b/eval/src/vespa/eval/eval/tensor_function.cpp @@ -350,12 +350,12 @@ Peek::make_spec() const // the value peeked is child 0, so // children (for label computation) in spec start at 1: size_t child_idx = 1; - for (const auto & [dim_name, label_or_child] : map()) { + for (const auto & [outer_dim_name, label_or_child] : map()) { std::visit(vespalib::overload { - [&,&dim_name = dim_name](const TensorSpec::Label &label) { + [&,&dim_name = outer_dim_name](const TensorSpec::Label &label) { generic_spec.emplace(dim_name, label); }, - [&,&dim_name = dim_name](const TensorFunction::Child &) { + [&,&dim_name = outer_dim_name](const TensorFunction::Child &) { generic_spec.emplace(dim_name, child_idx++); } }, label_or_child); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java index b16d26a04a4..bbf932ab652 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -22,6 +22,9 @@ public class FetchVector { * Note: If this enum is changed, you must also change {@link DimensionHelper}. */ public enum Dimension { + /** Application id from ApplicationId::toSerializedForm(TenantName, ApplicationName) on the form tenant:applicationName. */ + APPLICATION_ID, + /** * Cloud from com.yahoo.config.provision.CloudName::value, e.g. yahoo, aws, gcp. * diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 2e158f0f3ef..54a3ea4f2c2 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -13,6 +13,7 @@ import java.util.Optional; import java.util.TreeMap; import java.util.function.Predicate; +import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.INSTANCE_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.CLOUD_ACCOUNT; import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_ID; @@ -62,6 +63,7 @@ public class Flags { " latency-amortized-over-requests, latency-amortized-over-time", "Takes effect at redeployment (requires restart)", INSTANCE_ID); + public static final UnboundStringFlag SUMMARY_DECODE_POLICY = defineStringFlag( "summary-decode-policy", "eager", List.of("baldersheim"), "2023-03-30", "2023-12-31", @@ -311,10 +313,10 @@ public class Flags { "randomized-endpoint-names", false, List.of("andreer"), "2023-04-26", "2023-10-14", "Whether to use randomized endpoint names", "Takes effect on application deployment", - INSTANCE_ID); + INSTANCE_ID, APPLICATION_ID, TENANT_ID); public static final UnboundBooleanFlag ENABLE_THE_ONE_THAT_SHOULD_NOT_BE_NAMED = defineFeatureFlag( - "enable-the-one-that-should-not-be-named", false, List.of("hmusum"), "2023-05-08", "2023-10-01", + "enable-the-one-that-should-not-be-named", false, List.of("hmusum"), "2023-05-08", "2023-11-01", "Whether to enable the one program that should not be named", "Takes effect at next host-admin tick"); @@ -340,13 +342,13 @@ public class Flags { public static final UnboundBooleanFlag WRITE_CONFIG_SERVER_SESSION_DATA_AS_ONE_BLOB = defineFeatureFlag( "write-config-server-session-data-as-blob", false, - List.of("hmusum"), "2023-07-19", "2023-10-01", + List.of("hmusum"), "2023-07-19", "2023-11-01", "Whether to write config server session data in one blob or as individual paths", "Takes effect immediately"); public static final UnboundBooleanFlag READ_CONFIG_SERVER_SESSION_DATA_AS_ONE_BLOB = defineFeatureFlag( "read-config-server-session-data-as-blob", false, - List.of("hmusum"), "2023-07-19", "2023-10-01", + List.of("hmusum"), "2023-07-19", "2023-11-01", "Whether to read config server session data from session data blob or from individual paths", "Takes effect immediately"); @@ -371,12 +373,6 @@ public class Flags { "Takes effect on next host provisioning / run of host-admin", HOSTNAME, CLOUD_ACCOUNT); - public static final UnboundBooleanFlag WRITE_APPLICATION_DATA_AS_JSON = defineFeatureFlag( - "write-application-data-as-json", true, - List.of("hmusum"), "2023-08-27", "2023-10-01", - "Whether to write application data (active session id, last deployed session id etc. ) as json", - "Takes effect immediately"); - public static final UnboundIntFlag MIN_EXCLUSIVE_ADVERTISED_MEMORY_GB = defineIntFlag( "min-exclusive-advertised-memory-gb", 8, List.of("freva"), "2023-09-08", "2023-11-01", @@ -385,7 +381,7 @@ public class Flags { INSTANCE_ID, CLUSTER_ID, CLUSTER_TYPE); public static final UnboundBooleanFlag ASSIGN_RANDOMIZED_ID = defineFeatureFlag( - "assign-randomized-id", false, + "assign-randomized-id", true, List.of("mortent"), "2023-08-31", "2024-02-01", "Whether to assign randomized id to the application", "Takes effect immediately", @@ -406,6 +402,26 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); + public static final UnboundBooleanFlag DYNAMIC_HEAP_SIZE = defineFeatureFlag( + "dynamic-heap-size", false, + List.of("bjorncs"), "2023-09-21", "2024-01-15", + "Whether to calculate JVM heap size based on predicted Onnx model memory requirements", + "Takes effect at redeployment", + INSTANCE_ID); + + public static final UnboundStringFlag UNKNOWN_CONFIG_DEFINITION = defineStringFlag( + "unknown-config-definition", "warn", + List.of("hmusum"), "2023-09-25", "2023-11-01", + "How to handle user config referencing unknown config definitions. Valid values are log, warn, fail", + "Takes effect at redeployment", + INSTANCE_ID); + + public static final UnboundBooleanFlag LEGACY_ENDPOINTS = defineFeatureFlag( + "legacy-endpoints", true, List.of("mpolden", "tokle"), "2023-09-29", "2024-03-01", + "Whether legacy (non-anonymized) endpoints should be created in DNS", + "Takes effect on redeployment through controller", + INSTANCE_ID, APPLICATION_ID, TENANT_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java index f867daac245..04beec19d73 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java @@ -7,28 +7,30 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * @author hakonhall */ public class DimensionHelper { - private static final Map<FetchVector.Dimension, List<String>> serializedDimensions = new HashMap<>(); + private static final Map<FetchVector.Dimension, String> serializedDimensions = new HashMap<>(); static { - serializedDimensions.put(FetchVector.Dimension.CLOUD, List.of("cloud")); - serializedDimensions.put(FetchVector.Dimension.CLOUD_ACCOUNT, List.of("cloud-account")); - serializedDimensions.put(FetchVector.Dimension.CLUSTER_ID, List.of("cluster-id")); - serializedDimensions.put(FetchVector.Dimension.CLUSTER_TYPE, List.of("cluster-type")); - serializedDimensions.put(FetchVector.Dimension.CONSOLE_USER_EMAIL, List.of("console-user-email")); - serializedDimensions.put(FetchVector.Dimension.ENVIRONMENT, List.of("environment")); - serializedDimensions.put(FetchVector.Dimension.HOSTNAME, List.of("hostname")); - serializedDimensions.put(FetchVector.Dimension.INSTANCE_ID, List.of("application", "instance")); - serializedDimensions.put(FetchVector.Dimension.NODE_TYPE, List.of("node-type")); - serializedDimensions.put(FetchVector.Dimension.SYSTEM, List.of("system")); - serializedDimensions.put(FetchVector.Dimension.TENANT_ID, List.of("tenant")); - serializedDimensions.put(FetchVector.Dimension.VESPA_VERSION, List.of("vespa-version")); - serializedDimensions.put(FetchVector.Dimension.ZONE_ID, List.of("zone")); + serializedDimensions.put(FetchVector.Dimension.APPLICATION_ID, "application"); + serializedDimensions.put(FetchVector.Dimension.CLOUD, "cloud"); + serializedDimensions.put(FetchVector.Dimension.CLOUD_ACCOUNT, "cloud-account"); + serializedDimensions.put(FetchVector.Dimension.CLUSTER_ID, "cluster-id"); + serializedDimensions.put(FetchVector.Dimension.CLUSTER_TYPE, "cluster-type"); + serializedDimensions.put(FetchVector.Dimension.CONSOLE_USER_EMAIL, "console-user-email"); + serializedDimensions.put(FetchVector.Dimension.ENVIRONMENT, "environment"); + serializedDimensions.put(FetchVector.Dimension.HOSTNAME, "hostname"); + serializedDimensions.put(FetchVector.Dimension.INSTANCE_ID, "instance"); + serializedDimensions.put(FetchVector.Dimension.NODE_TYPE, "node-type"); + serializedDimensions.put(FetchVector.Dimension.SYSTEM, "system"); + serializedDimensions.put(FetchVector.Dimension.TENANT_ID, "tenant"); + serializedDimensions.put(FetchVector.Dimension.VESPA_VERSION, "vespa-version"); + serializedDimensions.put(FetchVector.Dimension.ZONE_ID, "zone"); if (serializedDimensions.size() != FetchVector.Dimension.values().length) { throw new IllegalStateException(FetchVectorHelper.class.getName() + " is not in sync with " + @@ -36,27 +38,16 @@ public class DimensionHelper { } } - private static final Map<String, FetchVector.Dimension> deserializedDimensions = reverseMapping(serializedDimensions); - - private static Map<String, FetchVector.Dimension> reverseMapping(Map<FetchVector.Dimension, List<String>> mapping) { - Map<String, FetchVector.Dimension> reverseMapping = new LinkedHashMap<>(); - mapping.forEach((dimension, serializedDimensions) -> { - serializedDimensions.forEach(serializedDimension -> { - if (reverseMapping.put(serializedDimension, dimension) != null) { - throw new IllegalStateException("Duplicate serialized dimension: '" + serializedDimension + "'"); - } - }); - }); - return Map.copyOf(reverseMapping); - } + private static final Map<String, FetchVector.Dimension> deserializedDimensions = serializedDimensions. + entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); public static String toWire(FetchVector.Dimension dimension) { - List<String> serializedDimension = serializedDimensions.get(dimension); - if (serializedDimension == null || serializedDimension.isEmpty()) { + String serializedDimension = serializedDimensions.get(dimension); + if (serializedDimension == null) { throw new IllegalArgumentException("Unsupported dimension (please add it): '" + dimension + "'"); } - return serializedDimension.get(0); + return serializedDimension; } public static FetchVector.Dimension fromWire(String serializedDimension) { diff --git a/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java b/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java index ed81afc8054..40310c47f78 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java @@ -30,7 +30,7 @@ public class FlagDataTest { }, { "type": "blacklist", - "dimension": "application", + "dimension": "instance", "values": [ "app1", "app2" ] } ], @@ -52,8 +52,6 @@ public class FlagDataTest { } }"""; - private final String json_with_instance = json.replace("application", "instance"); - private final FetchVector vector = new FetchVector(); @Test @@ -273,11 +271,6 @@ public class FlagDataTest { } private void verify(Optional<String> expectedValue, FetchVector vector) { - verify(json, expectedValue, vector); - verify(json_with_instance, expectedValue, vector); - } - - private void verify(String json, Optional<String> expectedValue, FetchVector vector) { FlagData data = FlagData.deserialize(json); assertEquals("id1", data.id().toString()); Optional<RawFlag> rawFlag = data.resolve(vector); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizer.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizer.java index 5df066f06fa..1a1e26b1091 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizer.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizer.java @@ -70,7 +70,7 @@ public class ExpressionOptimizer extends ExpressionConverter { } return exp instanceof InputExpression || exp instanceof NowExpression || - exp instanceof SetValueExpression || + exp instanceof ConstantExpression || exp instanceof HostNameExpression || exp instanceof GetVarExpression; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateAdapter.java index 3c3f75a6693..24ed6b898fd 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateAdapter.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateAdapter.java @@ -75,7 +75,7 @@ public class FieldPathUpdateAdapter implements UpdateAdapter { if (type == FieldPathEntry.Type.STRUCT_FIELD) { if (!(value instanceof StructuredFieldValue)) { throw new IllegalArgumentException("Expected structured field value, got " + - value.getClass().getName() + "."); + value.getClass().getName()); } for (Iterator<Map.Entry<Field, FieldValue>> it = ((StructuredFieldValue)value).iterator(); it.hasNext();) { Map.Entry<Field, FieldValue> structEntry = it.next(); @@ -84,8 +84,7 @@ public class FieldPathUpdateAdapter implements UpdateAdapter { createUpdatesAt(nextPath, structEntry.getValue(), idx + 1, out); } } else if (type == FieldPathEntry.Type.MAP_KEY) { - if (value instanceof WeightedSet) { - WeightedSet wset = (WeightedSet)value; + if (value instanceof WeightedSet wset) { for (Iterator<FieldValue> it = wset.fieldValueIterator(); it.hasNext();) { FieldValue wsetEntry = it.next(); List<FieldPathEntry> nextPath = new ArrayList<>(path); @@ -102,7 +101,7 @@ public class FieldPathUpdateAdapter implements UpdateAdapter { } } else { throw new IllegalArgumentException("Expected map or weighted set, got " + - value.getClass().getName() + "."); + value.getClass().getName()); } } else { path.add(pathEntry); @@ -111,7 +110,7 @@ public class FieldPathUpdateAdapter implements UpdateAdapter { } else if (update instanceof AddFieldPathUpdate) { if (!(value instanceof Array)) { throw new IllegalStateException("Expected array, got " + - value.getClass().getName() + "."); + value.getClass().getName()); } out.addFieldPathUpdate(new AddFieldPathUpdate(update.getDocumentType(), new FieldPath(path).toString(), update.getOriginalWhereClause(), (Array)value)); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateAdapter.java index 2601c5d0f71..b2ef838bcc4 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateAdapter.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateAdapter.java @@ -152,7 +152,7 @@ public class FieldUpdateAdapter implements UpdateAdapter { if (val instanceof Array) { lst.addAll(createMapValueUpdatesForArray((Array)val, (MapValueUpdate)upd)); } else if (val instanceof MapFieldValue) { - throw new UnsupportedOperationException("Can not map into a " + val.getClass().getName() + "."); + throw new UnsupportedOperationException("Can not map into a " + val.getClass().getName()); } else if (val instanceof StructuredFieldValue) { lst.addAll(createMapValueUpdatesForStruct((StructuredFieldValue)val, (MapValueUpdate)upd)); } else if (val instanceof WeightedSet) { @@ -168,7 +168,7 @@ public class FieldUpdateAdapter implements UpdateAdapter { lst.add(upd); } else { throw new UnsupportedOperationException( - "Value update type " + upd.getClass().getName() + " not supported."); + "Value update type " + upd.getClass().getName() + " not supported"); } return lst; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateHelper.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateHelper.java index 8c32f2e451d..a7fed91c360 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateHelper.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateHelper.java @@ -95,7 +95,7 @@ public abstract class FieldUpdateHelper { return applyUpdate(nestedUpdate, value); } } else if (value instanceof MapFieldValue) { - throw new UnsupportedOperationException("Can not map into a " + value.getClass().getName() + "."); + throw new UnsupportedOperationException("Can not map into a " + value.getClass().getName()); } else if (value instanceof StructuredFieldValue) { Field field = ((StructuredFieldValue)value).getField(String.valueOf(update.getValue())); if (field == null) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldValueConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldValueConverter.java index 4f96f2b7a31..d2cf97273ad 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldValueConverter.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldValueConverter.java @@ -49,7 +49,7 @@ public abstract class FieldValueConverter { nextType = nextVal.getDataType(); } else if (!nextType.isValueCompatible(nextVal)) { throw new IllegalArgumentException("Expected " + nextType.getName() + ", got " + - nextVal.getDataType().getName() + "."); + nextVal.getDataType().getName()); } next.add(nextVal); } @@ -63,7 +63,7 @@ public abstract class FieldValueConverter { @SuppressWarnings({ "unchecked", "rawtypes" }) private FieldValue convertMap(MapFieldValue<FieldValue, FieldValue> val) { - Map<FieldValue, FieldValue> next = new LinkedHashMap<FieldValue, FieldValue>(); + Map<FieldValue, FieldValue> next = new LinkedHashMap<>(); DataType nextKeyType = null, nextValType = null; for (Map.Entry<FieldValue, FieldValue> entry : val.entrySet()) { FieldValue prevKey = entry.getKey(); @@ -75,7 +75,7 @@ public abstract class FieldValueConverter { nextKeyType = nextKey.getDataType(); } else if (!nextKeyType.isValueCompatible(nextKey)) { throw new IllegalArgumentException("Expected " + nextKeyType.getName() + ", got " + - nextKey.getDataType().getName() + "."); + nextKey.getDataType().getName()); } FieldValue prevVal = entry.getValue(); FieldValue nextVal = convert(prevVal); @@ -86,7 +86,7 @@ public abstract class FieldValueConverter { nextValType = nextVal.getDataType(); } else if (!nextValType.isValueCompatible(nextVal)) { throw new IllegalArgumentException("Expected " + nextValType.getName() + ", got " + - nextVal.getDataType().getName() + "."); + nextVal.getDataType().getName()); } next.put(nextKey, nextVal); } @@ -100,7 +100,7 @@ public abstract class FieldValueConverter { @SuppressWarnings({ "unchecked", "rawtypes" }) private FieldValue convertWset(WeightedSet val) { - Map<FieldValue, Integer> next = new LinkedHashMap<FieldValue, Integer>(); + Map<FieldValue, Integer> next = new LinkedHashMap<>(); DataType nextType = null; for (Iterator<FieldValue> it = val.fieldValueIterator(); it.hasNext();) { FieldValue prevKey = it.next(); @@ -114,7 +114,7 @@ public abstract class FieldValueConverter { nextType = nextKey.getDataType(); } else if (!nextType.isValueCompatible(nextKey)) { throw new IllegalArgumentException("Expected " + nextType.getName() + ", got " + - nextKey.getDataType().getName() + "."); + nextKey.getDataType().getName()); } next.put(nextKey, prevVal); } @@ -143,7 +143,7 @@ public abstract class FieldValueConverter { } /** - * Returns whether or not the given {@link FieldValue} should be converted. If this method returns <em>false</em>, + * Returns whether the given {@link FieldValue} should be converted. If this method returns <em>false</em>, * the converter will proceed to traverse the value itself to see if its internal can be converted. * * @param value the value to check diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapter.java index f36c44539c7..bab1f5fe7c0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapter.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapter.java @@ -41,7 +41,7 @@ public class SimpleDocumentAdapter implements DocumentAdapter { try { return input.getDataType().buildFieldPath(fieldName).getResultingDataType(); } catch (IllegalArgumentException e) { - throw new VerificationException(exp, "Input field '" + fieldName + "' not found."); + throw new VerificationException(exp, "Input field '" + fieldName + "' not found"); } } @@ -67,12 +67,12 @@ public class SimpleDocumentAdapter implements DocumentAdapter { public void tryOutputType(Expression exp, String fieldName, DataType valueType) { Field field = output.getDataType().getField(fieldName); if (field == null) { - throw new VerificationException(exp, "Field '" + fieldName + "' not found."); + throw new VerificationException(exp, "Field '" + fieldName + "' not found"); } DataType fieldType = field.getDataType(); if (!fieldType.isAssignableFrom(valueType)) { throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to field '" + - fieldName + "' which is " + fieldType.getName() + "."); + fieldName + "' which is " + fieldType.getName()); } } @@ -81,7 +81,7 @@ public class SimpleDocumentAdapter implements DocumentAdapter { Field field = output.getField(fieldName); if (field == null) { throw new IllegalArgumentException("Field '" + fieldName + "' not found in document type '" + - output.getDataType().getName() + "'."); + output.getDataType().getName()); } output.setFieldValue(field, fieldValue); return this; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java index b7ee444975f..8fff5d488c2 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java @@ -42,28 +42,28 @@ public final class ArithmeticExpression extends CompositeExpression { } } - private final Expression lhs; + private final Expression left; private final Operator op; - private final Expression rhs; + private final Expression right; - public ArithmeticExpression(Expression lhs, Operator op, Expression rhs) { - super(requiredInputType(lhs, rhs)); - lhs.getClass(); // throws NullPointerException + public ArithmeticExpression(Expression left, Operator op, Expression right) { + super(requiredInputType(left, right)); + left.getClass(); // throws NullPointerException op.getClass(); - rhs.getClass(); - this.lhs = lhs; + right.getClass(); + this.left = left; this.op = op; - this.rhs = rhs; + this.right = right; } @Override public ArithmeticExpression convertChildren(ExpressionConverter converter) { // TODO: branch()? - return new ArithmeticExpression(converter.convert(lhs), op, converter.convert(rhs)); + return new ArithmeticExpression(converter.convert(left), op, converter.convert(right)); } public Expression getLeftHandSide() { - return lhs; + return left; } public Operator getOperator() { @@ -71,21 +71,21 @@ public final class ArithmeticExpression extends CompositeExpression { } public Expression getRightHandSide() { - return rhs; + return right; } @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); - context.setValue(evaluate(context.setValue(input).execute(lhs).getValue(), - context.setValue(input).execute(rhs).getValue())); + context.setValue(evaluate(context.setValue(input).execute(left).getValue(), + context.setValue(input).execute(right).getValue())); } @Override protected void doVerify(VerificationContext context) { DataType input = context.getValueType(); - context.setValueType(evaluate(context.setValueType(input).execute(lhs).getValueType(), - context.setValueType(input).execute(rhs).getValueType())); + context.setValueType(evaluate(context.setValueType(input).execute(left).getValueType(), + context.setValueType(input).execute(right).getValueType())); } private static DataType requiredInputType(Expression lhs, Expression rhs) { @@ -99,7 +99,7 @@ public final class ArithmeticExpression extends CompositeExpression { } if (!lhsType.equals(rhsType)) { throw new VerificationException(ArithmeticExpression.class, "Operands require conflicting input types, " + - lhsType.getName() + " vs " + rhsType.getName() + "."); + lhsType.getName() + " vs " + rhsType.getName()); } return lhsType; } @@ -111,7 +111,7 @@ public final class ArithmeticExpression extends CompositeExpression { @Override public String toString() { - return lhs + " " + op + " " + rhs; + return left + " " + op + " " + right; } @Override @@ -120,13 +120,13 @@ public final class ArithmeticExpression extends CompositeExpression { return false; } ArithmeticExpression exp = (ArithmeticExpression)obj; - if (!lhs.equals(exp.lhs)) { + if (!left.equals(exp.left)) { return false; } if (!op.equals(exp.op)) { return false; } - if (!rhs.equals(exp.rhs)) { + if (!right.equals(exp.right)) { return false; } return true; @@ -134,12 +134,12 @@ public final class ArithmeticExpression extends CompositeExpression { @Override public int hashCode() { - return getClass().hashCode() + lhs.hashCode() + op.hashCode() + rhs.hashCode(); + return getClass().hashCode() + left.hashCode() + op.hashCode() + right.hashCode(); } private DataType evaluate(DataType lhs, DataType rhs) { if (lhs == null || rhs == null) { - throw new VerificationException(this, "Attempting to perform arithmetic on a null value."); + throw new VerificationException(this, "Attempting to perform arithmetic on a null value"); } if (!(lhs instanceof NumericDataType) || !(rhs instanceof NumericDataType)) @@ -210,12 +210,12 @@ public final class ArithmeticExpression extends CompositeExpression { return BigDecimal.valueOf(((LongFieldValue)value).getLong()); } throw new IllegalArgumentException("Unsupported numeric field value type '" + - value.getClass().getName() + "'."); + value.getClass().getName() + "'"); } @Override public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { - lhs.select(predicate, operation); - rhs.select(predicate, operation); + left.select(predicate, operation); + right.select(predicate, operation); } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java index 2474dadab77..d5b5fa2ddfa 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java @@ -23,11 +23,11 @@ public final class Base64DecodeExpression extends Expression { return; } if (input.length() > 12) { - throw new NumberFormatException("Base64 value '" + input + "' is out of range."); + throw new NumberFormatException("Base64 value '" + input + "' is out of range"); } byte[] decoded = Base64.getDecoder().decode(input); if (decoded == null || decoded.length == 0) { - throw new NumberFormatException("Illegal base64 value '" + input + "'."); + throw new NumberFormatException("Illegal base64 value '" + input + "'"); } long output = 0; for (int i = decoded.length; --i >= 0;) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java index 564ab015e10..b495a0b3bbf 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java @@ -62,7 +62,7 @@ public final class CatExpression extends ExpressionList<Expression> { DataType val = context.setValueType(input).execute(exp).getValueType(); types.add(val); if (val == null) { - throw new VerificationException(this, "Attempting to concatenate a null value (" + exp + ")."); + throw new VerificationException(this, "Attempting to concatenate a null value (" + exp + ")"); } } context.setValueType(resolveOutputType(types)); @@ -78,7 +78,7 @@ public final class CatExpression extends ExpressionList<Expression> { prev = next; } else if (!prev.isAssignableFrom(next)) { throw new VerificationException(CatExpression.class, "Operands require conflicting input types, " + - prev.getName() + " vs " + next.getName() + "."); + prev.getName() + " vs " + next.getName()); } } return prev; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java index 5dbb9292a9d..991cbd30433 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java @@ -64,14 +64,14 @@ public class ChoiceExpression extends ExpressionList<Expression> { previousInput = thisInput; else if (thisInput != null && !previousInput.isAssignableFrom(thisInput)) throw new VerificationException(ScriptExpression.class, "Choice expression require conflicting input types, " + - previousInput.getName() + " vs " + thisInput.getName() + "."); + previousInput.getName() + " vs " + thisInput.getName()); DataType thisOutput = choice.createdOutputType(); if (previousOutput == null) previousOutput = thisOutput; else if (thisOutput != null && !previousOutput.isAssignableFrom(thisOutput)) throw new VerificationException(ScriptExpression.class, "Choice expression produce conflicting output types, " + - previousOutput.getName() + " vs " + thisOutput.getName() + "."); + previousOutput.getName() + " vs " + thisOutput.getName()); } return previousInput; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ConstantExpression.java index f7348c24af5..b44d4844c4d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ConstantExpression.java @@ -7,17 +7,18 @@ import com.yahoo.document.datatypes.LongFieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.text.StringUtilities; +import java.util.Objects; + /** * @author Simon Thoresen Hult */ -public final class SetValueExpression extends Expression { +public final class ConstantExpression extends Expression { private final FieldValue value; - public SetValueExpression(FieldValue value) { + public ConstantExpression(FieldValue value) { super(null); - value.getClass(); // throws NullPointerException - this.value = value; + this.value = Objects.requireNonNull(value); } public FieldValue getValue() { @@ -52,7 +53,7 @@ public final class SetValueExpression extends Expression { @Override public boolean equals(Object obj) { - if (!(obj instanceof SetValueExpression rhs)) return false; + if (!(obj instanceof ConstantExpression rhs)) return false; if (!value.equals(rhs.value)) return false; return true; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java index 5ee5fea3158..0407e17596b 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java @@ -116,7 +116,7 @@ public class EmbedExpression extends Expression { String outputField = context.getOutputField(); if (outputField == null) throw new VerificationException(this, "No output field in this statement: " + - "Don't know what tensor type to embed into."); + "Don't know what tensor type to embed into"); targetType = toTargetTensor(context.getInputType(this, outputField)); if ( ! validTarget(targetType)) throw new VerificationException(this, "The embedding target field must either be a dense 1d tensor, " + diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java index f01f2fcc9fb..7f0abfd64db 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java @@ -50,7 +50,7 @@ public class ExecutionContext implements FieldTypeAdapter, FieldValueAdapter, Cl @Override public FieldValue getInputValue(String fieldName) { if (adapter == null) { - throw new IllegalStateException("Can not get field '" + fieldName + "' because adapter is null."); + throw new IllegalStateException("Can not get field '" + fieldName + "' because adapter is null"); } return adapter.getInputValue(fieldName); } @@ -58,7 +58,7 @@ public class ExecutionContext implements FieldTypeAdapter, FieldValueAdapter, Cl @Override public FieldValue getInputValue(FieldPath fieldPath) { if (adapter == null) { - throw new IllegalStateException("Can not get field '" + fieldPath + "' because adapter is null."); + throw new IllegalStateException("Can not get field '" + fieldPath + "' because adapter is null"); } return adapter.getInputValue(fieldPath); } @@ -71,7 +71,7 @@ public class ExecutionContext implements FieldTypeAdapter, FieldValueAdapter, Cl @Override public ExecutionContext setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) { if (adapter == null) - throw new IllegalStateException("Can not set field '" + fieldName + "' because adapter is null."); + throw new IllegalStateException("Can not set field '" + fieldName + "' because adapter is null"); adapter.setOutputValue(exp, fieldName, fieldValue); return this; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java index f498b871096..a6f117beb40 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java @@ -28,8 +28,8 @@ public abstract class Expression extends Selectable { * Creates an expression * * @param inputType the type of the input this expression can work with. - * UnresolvedDataType.INSTANCE if it works with any type, - * and null if it does not consume any input. + * UnresolvedDataType.INSTANCE if it works with any type, + * and null if it does not consume any input. */ protected Expression(DataType inputType) { this.inputType = inputType; @@ -93,7 +93,7 @@ public abstract class Expression extends Selectable { } if (!inputType.isValueCompatible(input)) { throw new IllegalArgumentException("Expression '" + this + "' expected " + inputType.getName() + - " input, got " + input.getDataType().getName() + "."); + " input, got " + input.getDataType().getName()); } } doExecute(context); @@ -102,7 +102,7 @@ public abstract class Expression extends Selectable { FieldValue output = context.getValue(); if (output != null && !outputType.isValueCompatible(output)) { throw new IllegalStateException("Expression '" + this + "' expected " + outputType.getName() + - " output, got " + output.getDataType().getName() + "."); + " output, got " + output.getDataType().getName()); } } return context.getValue(); @@ -163,14 +163,14 @@ public abstract class Expression extends Selectable { if (inputType != null) { DataType input = context.getValueType(); if (input == null) { - throw new VerificationException(this, "Expected " + inputType.getName() + " input, got null."); + throw new VerificationException(this, "Expected " + inputType.getName() + " input, but no input is specified"); } if (input.getPrimitiveType() == UnresolvedDataType.INSTANCE) { - throw new VerificationException(this, "Failed to resolve input type."); + throw new VerificationException(this, "Failed to resolve input type"); } if (!inputType.isAssignableFrom(input)) { throw new VerificationException(this, "Expected " + inputType.getName() + " input, got " + - input.getName() + "."); + input.getName()); } } doVerify(context); @@ -178,14 +178,13 @@ public abstract class Expression extends Selectable { if (outputType != null) { DataType output = context.getValueType(); if (output == null) { - throw new VerificationException(this, "Expected " + outputType.getName() + " output, got null."); + throw new VerificationException(this, "Expected " + outputType.getName() + " output, but no output is specified"); } if (output.getPrimitiveType() == UnresolvedDataType.INSTANCE) { - throw new VerificationException(this, "Failed to resolve output type."); + throw new VerificationException(this, "Failed to resolve output type"); } if (!outputType.isAssignableFrom(output)) { - throw new VerificationException(this, "Expected " + outputType.getName() + " output, got " + - output.getName() + "."); + throw new VerificationException(this, "Expected " + outputType.getName() + " output, got " + output.getName()); } } return context.getValueType(); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java index 57de66f80a0..f3e9e33d841 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java @@ -21,9 +21,9 @@ public abstract class ExpressionList<T extends Expression> extends CompositeExpr private final List<T> expressions = new LinkedList<T>(); - protected ExpressionList(Iterable<? extends T> lst, DataType inputType) { + protected ExpressionList(Iterable<? extends T> expressions, DataType inputType) { super(inputType); - for (T exp : lst) { + for (T exp : expressions) { this.expressions.add(exp); } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java index 3053a391823..7e32c93faff 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java @@ -52,7 +52,7 @@ public final class ForEachExpression extends CompositeExpression { context.setValue(new MyConverter(context, exp).convert(input)); } else { throw new IllegalArgumentException("Expected Array, Struct or WeightedSet input, got " + - input.getDataType().getName() + "."); + input.getDataType().getName()); } } @@ -80,13 +80,13 @@ public final class ForEachExpression extends CompositeExpression { DataType structValueType = context.setValueType(fieldType).execute(exp).getValueType(); if (!fieldType.isAssignableFrom(structValueType)) throw new VerificationException(this, "Expected " + fieldType.getName() + " output, got " + - structValueType.getName() + "."); + structValueType.getName()); } context.setValueType(valueType); } else { throw new VerificationException(this, "Expected Array, Struct or WeightedSet input, got " + - valueType.getName() + "."); + valueType.getName()); } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java index ecb6980f795..8a0fc9a56ec 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java @@ -27,7 +27,7 @@ public final class GetFieldExpression extends Expression { protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); if (!(input instanceof StructuredFieldValue struct)) { - throw new IllegalArgumentException("Expected structured input, got " + input.getDataType().getName() + "."); + throw new IllegalArgumentException("Expected structured input, got " + input.getDataType().getName()); } Field field = struct.getField(fieldName); if (field == null) { @@ -41,7 +41,7 @@ public final class GetFieldExpression extends Expression { protected void doVerify(VerificationContext context) { DataType input = context.getValueType(); if (!(input instanceof StructuredDataType)) { - throw new VerificationException(this, "Expected structured input, got " + input.getName() + "."); + throw new VerificationException(this, "Expected structured input, got " + input.getName()); } Field field = ((StructuredDataType)input).getField(fieldName); if (field == null) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java index 4ebf5da2ff8..54e85be1986 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java @@ -28,7 +28,7 @@ public final class GetVarExpression extends Expression { protected void doVerify(VerificationContext context) { DataType input = context.getVariable(varName); if (input == null) { - throw new VerificationException(this, "Variable '" + varName + "' not found."); + throw new VerificationException(this, "Variable '" + varName + "' not found"); } context.setValueType(input); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HashExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HashExpression.java index 2952692b5d0..3b4c1b432bf 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HashExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HashExpression.java @@ -63,7 +63,7 @@ public class HashExpression extends Expression { String outputField = context.getOutputField(); if (outputField == null) throw new VerificationException(this, "No output field in this statement: " + - "Don't know what value to hash to."); + "Don't know what value to hash to"); DataType outputFieldType = context.getInputType(this, outputField); if ( ! canStoreHash(outputFieldType)) throw new VerificationException(this, "The type of the output field " + outputField + diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java index 93f101a422e..4a2c7381ac0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java @@ -28,12 +28,12 @@ public final class HexDecodeExpression extends Expression { try { output = new BigInteger(input, 16); } catch (NumberFormatException e) { - throw new NumberFormatException("Illegal hex value '" + input + "'."); + throw new NumberFormatException("Illegal hex value '" + input + "'"); } if (output.bitLength() > 64) { - throw new NumberFormatException("Hex value '" + input + "' is out of range."); + throw new NumberFormatException("Hex value '" + input + "' is out of range"); } - if (output.compareTo(BigInteger.ZERO) == 1 && output.bitLength() == 64) { + if (output.compareTo(BigInteger.ZERO) > 0 && output.bitLength() == 64) { output = output.subtract(ULONG_MAX); // flip to negative } context.setValue(new LongFieldValue(output.longValue())); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java index f05795aa234..e0fb4e0337a 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java @@ -27,7 +27,7 @@ public final class IfThenExpression extends CompositeExpression { private final String img; - private Comparator(String img) { + Comparator(String img) { this.img = img; } @@ -38,77 +38,67 @@ public final class IfThenExpression extends CompositeExpression { } - private final Expression lhs; - private final Comparator cmp; - private final Expression rhs; + private final Expression left; + private final Comparator comparator; + private final Expression right; private final Expression ifTrue; private final Expression ifFalse; - public IfThenExpression(Expression lhs, Comparator cmp, Expression rhs, Expression ifTrue) { - this(lhs, cmp, rhs, ifTrue, null); + public IfThenExpression(Expression lhs, Comparator cmp, Expression right, Expression ifTrue) { + this(lhs, cmp, right, ifTrue, null); } - public IfThenExpression(Expression lhs, Comparator cmp, Expression rhs, Expression ifTrue, Expression ifFalse) { - super(resolveInputType(lhs, rhs, ifTrue, ifFalse)); - this.lhs = lhs; - this.cmp = cmp; - this.rhs = rhs; + public IfThenExpression(Expression lhs, Comparator cmp, Expression right, Expression ifTrue, Expression ifFalse) { + super(resolveInputType(lhs, right, ifTrue, ifFalse)); + this.left = lhs; + this.comparator = cmp; + this.right = right; this.ifTrue = ifTrue; this.ifFalse = ifFalse; } @Override public IfThenExpression convertChildren(ExpressionConverter converter) { - return new IfThenExpression(converter.branch().convert(lhs), - cmp, - converter.branch().convert(rhs), + return new IfThenExpression(converter.branch().convert(left), + comparator, + converter.branch().convert(right), converter.branch().convert(ifTrue), converter.branch().convert(ifFalse)); } @Override public void setStatementOutput(DocumentType documentType, Field field) { - lhs.setStatementOutput(documentType, field); - rhs.setStatementOutput(documentType, field); + left.setStatementOutput(documentType, field); + right.setStatementOutput(documentType, field); ifTrue.setStatementOutput(documentType, field); ifFalse.setStatementOutput(documentType, field); } - public Expression getLeftHandSide() { - return lhs; - } + public Expression getLeftHandSide() { return left; } - public Comparator getComparator() { - return cmp; - } + public Comparator getComparator() { return comparator; } - public Expression getRightHandSide() { - return rhs; - } + public Expression getRightHandSide() { return right; } - public Expression getIfTrueExpression() { - return ifTrue; - } + public Expression getIfTrueExpression() { return ifTrue; } - public Expression getIfFalseExpression() { - return ifFalse; - } + public Expression getIfFalseExpression() { return ifFalse; } @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); - FieldValue lhsVal = context.setValue(input).execute(lhs).getValue(); - if (lhsVal == null) { + FieldValue leftValue = context.setValue(input).execute(left).getValue(); + if (leftValue == null) { context.setValue(null); return; } - FieldValue rhsVal = context.setValue(input).execute(rhs).getValue(); - if (rhsVal == null) { + FieldValue rightValue = context.setValue(input).execute(right).getValue(); + if (rightValue == null) { context.setValue(null); return; } context.setValue(input); - if (isTrue(lhsVal, cmp, rhsVal)) { + if (isTrue(leftValue, comparator, rightValue)) { ifTrue.execute(context); } else if (ifFalse != null) { ifFalse.execute(context); @@ -118,17 +108,19 @@ public final class IfThenExpression extends CompositeExpression { @Override protected void doVerify(VerificationContext context) { DataType input = context.getValueType(); - context.setValueType(input).execute(lhs); - context.setValueType(input).execute(rhs); - context.setValueType(input).execute(ifTrue); - context.setValueType(input).execute(ifFalse); - context.setValueType(input); + context.setValueType(input).execute(left); + context.setValueType(input).execute(right); + var trueValue = context.setValueType(input).execute(ifTrue); + var falseValue = context.setValueType(input).execute(ifFalse); + var valueType = trueValue.getValueType().isAssignableFrom(falseValue.getValueType()) ? + trueValue.getValueType() : falseValue.getValueType(); + context.setValueType(valueType); } @Override public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { - select(lhs, predicate, operation); - select(rhs, predicate, operation); + select(left, predicate, operation); + select(right, predicate, operation); select(ifTrue, predicate, operation); select(ifFalse, predicate, operation); } @@ -146,13 +138,19 @@ public final class IfThenExpression extends CompositeExpression { @Override public DataType createdOutputType() { - return null; + DataType ifTrueType = ifTrue.createdOutputType(); + DataType ifFalseType = ifFalse == null ? null : ifFalse.createdOutputType(); + if (ifTrueType == null || ifFalseType == null) return null; + if (ifTrueType.isAssignableFrom(ifFalseType)) + return ifTrueType; + else + return ifFalseType; } @Override public String toString() { StringBuilder ret = new StringBuilder(); - ret.append("if (").append(lhs).append(" ").append(cmp).append(" ").append(rhs).append(") "); + ret.append("if (").append(left).append(" ").append(comparator).append(" ").append(right).append(") "); ret.append(toScriptBlock(ifTrue)); if (ifFalse != null) { ret.append(" else ").append(toScriptBlock(ifFalse)); @@ -165,13 +163,13 @@ public final class IfThenExpression extends CompositeExpression { if (!(obj instanceof IfThenExpression exp)) { return false; } - if (!lhs.equals(exp.lhs)) { + if (!left.equals(exp.left)) { return false; } - if (!cmp.equals(exp.cmp)) { + if (!comparator.equals(exp.comparator)) { return false; } - if (!rhs.equals(exp.rhs)) { + if (!right.equals(exp.right)) { return false; } if (!ifTrue.equals(exp.ifTrue)) { @@ -185,7 +183,7 @@ public final class IfThenExpression extends CompositeExpression { @Override public int hashCode() { - int ret = getClass().hashCode() + lhs.hashCode() + cmp.hashCode() + rhs.hashCode() + ifTrue.hashCode(); + int ret = getClass().hashCode() + left.hashCode() + comparator.hashCode() + right.hashCode() + ifTrue.hashCode(); if (ifFalse != null) { ret += ifFalse.hashCode(); } @@ -201,7 +199,7 @@ public final class IfThenExpression extends CompositeExpression { } if (!prev.equals(next)) { throw new VerificationException(IfThenExpression.class, "Operands require conflicting input types, " + - prev.getName() + " vs " + next.getName() + "."); + prev.getName() + " vs " + next.getName()); } return prev; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java index bba1b09cda2..8534e58694e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java @@ -42,7 +42,7 @@ public final class InputExpression extends Expression { protected void doVerify(VerificationContext context) { DataType val = context.getInputType(this, fieldName); if (val == null) - throw new VerificationException(this, "Field '" + fieldName + "' not found."); + throw new VerificationException(this, "Field '" + fieldName + "' not found"); context.setValueType(val); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java index 1c3582ea695..39325385dde 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java @@ -31,7 +31,7 @@ public final class JoinExpression extends Expression { protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); if (!(input instanceof Array)) { - throw new IllegalArgumentException("Expected Array input, got " + input.getDataType().getName() + "."); + throw new IllegalArgumentException("Expected Array input, got " + input.getDataType().getName()); } StringBuilder output = new StringBuilder(); for (Iterator<FieldValue> it = ((Array)input).fieldValueIterator(); it.hasNext(); ) { @@ -47,7 +47,7 @@ public final class JoinExpression extends Expression { protected void doVerify(VerificationContext context) { DataType input = context.getValueType(); if (!(input instanceof ArrayDataType)) { - throw new VerificationException(this, "Expected Array input, got " + input.getName() + "."); + throw new VerificationException(this, "Expected Array input, got " + input.getName()); } context.setValueType(createdOutputType()); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java index d4b6f2e0a0a..97bbb1494e0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java @@ -54,10 +54,10 @@ public final class OptimizePredicateExpression extends Expression { DataType input = ctx.getVariable(var); if (input == null) { if (required) { - throw new VerificationException(this, "Variable '" + var + "' must be set."); + throw new VerificationException(this, "Variable '" + var + "' must be set"); } } else if (input != type) { - throw new VerificationException(this, "Variable '" + var + "' must have type " + type.getName() + "."); + throw new VerificationException(this, "Variable '" + var + "' must have type " + type.getName()); } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java index 1a640c9924e..f9ceed4cb34 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java @@ -29,12 +29,12 @@ public final class ScriptExpression extends ExpressionList<StatementExpression> this(Collections.emptyList()); } - public ScriptExpression(StatementExpression... lst) { - this(Arrays.asList(lst)); + public ScriptExpression(StatementExpression... statements) { + this(Arrays.asList(statements)); } - public ScriptExpression(Collection<? extends StatementExpression> lst) { - super(lst, resolveInputType(lst)); + public ScriptExpression(Collection<? extends StatementExpression> statements) { + super(statements, resolveInputType(statements)); } @Override @@ -67,10 +67,8 @@ public final class ScriptExpression extends ExpressionList<StatementExpression> @Override protected void doVerify(VerificationContext context) { DataType input = context.getValueType(); - for (Expression exp : this) { + for (Expression exp : this) context.setValueType(input).execute(exp); - } - context.setValueType(input); } private static DataType resolveInputType(Collection<? extends StatementExpression> list) { @@ -81,7 +79,7 @@ public final class ScriptExpression extends ExpressionList<StatementExpression> prev = next; } else if (next != null && !prev.isAssignableFrom(next)) { throw new VerificationException(ScriptExpression.class, "Statements require conflicting input types, " + - prev.getName() + " vs " + next.getName() + "."); + prev.getName() + " vs " + next.getName()); } } return prev; @@ -89,7 +87,9 @@ public final class ScriptExpression extends ExpressionList<StatementExpression> @Override public DataType createdOutputType() { - return null; + var expressions = asList(); + if (expressions.isEmpty()) return null; + return (expressions.get(expressions.size() - 1)).createdOutputType(); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java index bb8111f358e..90b2253b520 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java @@ -65,7 +65,7 @@ public final class SelectInputExpression extends CompositeExpression { for (Pair<String, Expression> entry : cases) { DataType val = context.getInputType(this, entry.getFirst()); if (val == null) { - throw new VerificationException(this, "Field '" + entry.getFirst() + "' not found."); + throw new VerificationException(this, "Field '" + entry.getFirst() + "' not found"); } context.setValueType(val).execute(entry.getSecond()); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java index a855ba86c9c..c80efbf7d19 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java @@ -30,7 +30,7 @@ public final class SetVarExpression extends Expression { DataType prev = context.getVariable(varName); if (prev != null && !prev.equals(next)) { throw new VerificationException(this, "Attempting to assign conflicting types to variable '" + varName + - "', " + prev.getName() + " vs " + next.getName() + "."); + "', " + prev.getName() + " vs " + next.getName()); } context.setVariable(varName, next); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java index 75f206ef47d..da067935d18 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java @@ -73,12 +73,12 @@ public final class StatementExpression extends ExpressionList<Expression> { context.execute(expression); } - private static DataType resolveInputType(Iterable<Expression> lst) { - for (Expression exp : lst) { - DataType type = exp.requiredInputType(); + private static DataType resolveInputType(Iterable<Expression> expressions) { + for (Expression expression : expressions) { + DataType type = expression.requiredInputType(); if (type != null) return type; - type = exp.createdOutputType(); + type = expression.createdOutputType(); if (type != null) return null; } return null; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java index c7cf7066483..3bf67ff9c5d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java @@ -70,7 +70,7 @@ public final class SwitchExpression extends CompositeExpression { if (input != null) { if (!(input instanceof StringFieldValue)) { throw new IllegalArgumentException("Expected " + DataType.STRING.getName() + " input, got " + - input.getDataType().getName() + "."); + input.getDataType().getName()); } exp = cases.get(String.valueOf(input)); } @@ -95,11 +95,11 @@ public final class SwitchExpression extends CompositeExpression { protected void doVerify(VerificationContext context) { DataType input = context.getValueType(); if (input == null) { - throw new VerificationException(this, "Expected " + DataType.STRING.getName() + " input, got null."); + throw new VerificationException(this, "Expected " + DataType.STRING.getName() + " input, but no input is specified"); } if (input != DataType.STRING) { throw new VerificationException(this, "Expected " + DataType.STRING.getName() + " input, got " + - input.getName() + "."); + input.getName()); } for (Expression exp : cases.values()) { context.setValueType(input).execute(exp); diff --git a/indexinglanguage/src/main/javacc/IndexingParser.jj b/indexinglanguage/src/main/javacc/IndexingParser.jj index d559d9b7260..3c67a468aea 100644 --- a/indexinglanguage/src/main/javacc/IndexingParser.jj +++ b/indexinglanguage/src/main/javacc/IndexingParser.jj @@ -32,7 +32,6 @@ import com.yahoo.document.datatypes.*; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.indexinglanguage.expressions.*; import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; -import com.yahoo.language.process.StemMode; import com.yahoo.language.process.Embedder; import com.yahoo.language.Linguistics; @@ -592,7 +591,7 @@ Expression setValueExp() : } { ( val = fieldValue() ) - { return new SetValueExpression(val); } + { return new ConstantExpression(val); } } Expression setVarExp() : diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java index 8aeaa084e1b..329ea6c95ba 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.Base64DecodeExpression; import com.yahoo.vespa.indexinglanguage.expressions.Base64EncodeExpression; import com.yahoo.vespa.indexinglanguage.expressions.CatExpression; import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ConstantExpression; import com.yahoo.vespa.indexinglanguage.expressions.EchoExpression; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; @@ -32,7 +33,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.RandomExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.indexinglanguage.expressions.SelectInputExpression; import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression; import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; import com.yahoo.vespa.indexinglanguage.expressions.SplitExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; @@ -103,7 +103,7 @@ public class ExpressionConverterTestCase { assertConvertable(new SelectInputExpression(new Pair<String, Expression>("foo", new IndexExpression("bar")), new Pair<String, Expression>("bar", new IndexExpression("foo")))); assertConvertable(new SetLanguageExpression()); - assertConvertable(new SetValueExpression(new IntegerFieldValue(69))); + assertConvertable(new ConstantExpression(new IntegerFieldValue(69))); assertConvertable(new SetVarExpression("foo")); assertConvertable(new SplitExpression("foo")); assertConvertable(new StatementExpression(new InputExpression("foo"))); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizerTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizerTestCase.java index f16e170c981..d9cb89d6be5 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizerTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizerTestCase.java @@ -1,15 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.indexinglanguage; -import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.vespa.indexinglanguage.expressions.*; import com.yahoo.vespa.indexinglanguage.parser.ParseException; import org.junit.Test; -import java.util.HashMap; -import java.util.Map; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -26,7 +22,7 @@ public class ExpressionOptimizerTestCase { public void requireThatStatementsBeforeOneThatIgnoresInputAreRemoved() { checkStatementThatIgnoresInput(new InputExpression("foo")); checkStatementThatIgnoresInput(new NowExpression()); - checkStatementThatIgnoresInput(new SetValueExpression(new IntegerFieldValue(42))); + checkStatementThatIgnoresInput(new ConstantExpression(new IntegerFieldValue(42))); checkStatementThatIgnoresInput(new HostNameExpression()); checkStatementThatIgnoresInput(new RandomExpression(42)); checkStatementThatIgnoresInput(new ArithmeticExpression( diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java index 8683874d44d..84e5c730fa6 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.indexinglanguage; import com.yahoo.collections.Pair; import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression; import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; @@ -11,6 +10,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.Base64DecodeExpression; import com.yahoo.vespa.indexinglanguage.expressions.Base64EncodeExpression; import com.yahoo.vespa.indexinglanguage.expressions.CatExpression; import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ConstantExpression; import com.yahoo.vespa.indexinglanguage.expressions.EchoExpression; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; @@ -33,7 +33,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.RandomExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.indexinglanguage.expressions.SelectInputExpression; import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression; import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; import com.yahoo.vespa.indexinglanguage.expressions.SplitExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; @@ -100,7 +99,7 @@ public class ExpressionVisitorTestCase { assertCount(3, new SelectInputExpression(new Pair<String, Expression>("foo", new IndexExpression("bar")), new Pair<String, Expression>("bar", new IndexExpression("foo")))); assertCount(1, new SetLanguageExpression()); - assertCount(1, new SetValueExpression(new IntegerFieldValue(69))); + assertCount(1, new ConstantExpression(new IntegerFieldValue(69))); assertCount(1, new SetVarExpression("foo")); assertCount(1, new SplitExpression("foo")); assertCount(2, new StatementExpression(new InputExpression("foo"))); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java index 98458dd965c..77b38dd7549 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java @@ -69,7 +69,7 @@ public class ScriptTestCase { fail(); } catch (VerificationException e) { assertEquals(e.getExpressionType(), ScriptExpression.class); - assertEquals("Expected any input, got null.", e.getMessage()); + assertEquals("Expected any input, but no input is specified", e.getMessage()); } } @@ -85,7 +85,7 @@ public class ScriptTestCase { } @Test - public void requireThatIfExpressionPassesOriginalInputAlong() throws ParseException { + public void requireThatIfExpressionReturnsTheProducedType() throws ParseException { Document input = new Document(type, "id:scheme:mytype::"); Document output = Expression.execute(Expression.fromString("'foo' | if (1 < 2) { 'bar' | index 'out-1' } else { 'baz' | index 'out-1' } | index 'out-1'"), input); assertNotNull(output); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapterTestCase.java index 76f96a80bb9..5a85d8ef5c3 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapterTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapterTestCase.java @@ -48,7 +48,7 @@ public class SimpleDocumentAdapterTestCase { adapter.getInputType(null, "foo"); fail(); } catch (VerificationException e) { - assertEquals("Input field 'foo' not found.", e.getMessage()); + assertEquals("Input field 'foo' not found", e.getMessage()); } assertNull(adapter.getInputValue("foo")); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueTransformProviderTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueTransformProviderTestCase.java index d0e9356ee76..f7eeca6f70a 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueTransformProviderTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueTransformProviderTestCase.java @@ -111,14 +111,14 @@ public class ValueTransformProviderTestCase { @Test public void requireThatSelectInputBranchesAreManagedSeparately() { - List<Pair<String, Expression>> before = new LinkedList<Pair<String, Expression>>(); - before.add(new Pair<String, Expression>("a", new IndexExpression("b"))); - before.add(new Pair<String, Expression>("c", new IndexExpression("d"))); + List<Pair<String, Expression>> before = new LinkedList<>(); + before.add(new Pair<>("a", new IndexExpression("b"))); + before.add(new Pair<>("c", new IndexExpression("d"))); - List<Pair<String, Expression>> after = new LinkedList<Pair<String, Expression>>(); - after.add(new Pair<String, Expression>("a", new StatementExpression(new LowerCaseExpression(), + List<Pair<String, Expression>> after = new LinkedList<>(); + after.add(new Pair<>("a", new StatementExpression(new LowerCaseExpression(), new IndexExpression("b")))); - after.add(new Pair<String, Expression>("c", new StatementExpression(new LowerCaseExpression(), + after.add(new Pair<>("c", new StatementExpression(new LowerCaseExpression(), new IndexExpression("d")))); assertProvided(new StatementExpression(new SelectInputExpression(before), @@ -130,11 +130,11 @@ public class ValueTransformProviderTestCase { @Test public void requireThatSwitchBranchesAreManagedSeparately() { - Map<String, Expression> before = new LinkedHashMap<String, Expression>(); + Map<String, Expression> before = new LinkedHashMap<>(); before.put("a", new IndexExpression("b")); before.put("c", new IndexExpression("d")); - Map<String, Expression> after = new LinkedHashMap<String, Expression>(); + Map<String, Expression> after = new LinkedHashMap<>(); after.put("a", new StatementExpression(new LowerCaseExpression(), new IndexExpression("b"))); after.put("c", new StatementExpression(new LowerCaseExpression(), @@ -170,4 +170,5 @@ public class ValueTransformProviderTestCase { return new LowerCaseExpression(); } } + } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java index 91d8f833701..f6dc7f839ed 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java @@ -64,13 +64,13 @@ public class ArithmeticTestCase { SimpleExpression.newOutput(DataType.INT), null); assertVerifyThrows(SimpleExpression.newOutput(null), Operator.ADD, SimpleExpression.newOutput(DataType.INT), null, - "Attempting to perform arithmetic on a null value."); + "Attempting to perform arithmetic on a null value"); assertVerifyThrows(SimpleExpression.newOutput(DataType.INT), Operator.ADD, SimpleExpression.newOutput(null), null, - "Attempting to perform arithmetic on a null value."); + "Attempting to perform arithmetic on a null value"); assertVerifyThrows(SimpleExpression.newOutput(null), Operator.ADD, SimpleExpression.newOutput(null), null, - "Attempting to perform arithmetic on a null value."); + "Attempting to perform arithmetic on a null value"); assertVerifyThrows(SimpleExpression.newOutput(DataType.INT), Operator.ADD, SimpleExpression.newOutput(DataType.STRING), null, "Attempting to perform unsupported arithmetic: [int] + [string]"); @@ -95,7 +95,7 @@ public class ArithmeticTestCase { new SimpleExpression(DataType.INT), DataType.INT); assertVerifyThrows(new SimpleExpression(DataType.INT), Operator.ADD, new SimpleExpression(DataType.STRING), null, - "Operands require conflicting input types, int vs string."); + "Operands require conflicting input types, int vs string"); } @Test @@ -114,23 +114,23 @@ public class ArithmeticTestCase { @Test public void requireThatArithmeticWithNullEvaluatesToNull() { assertNull(newArithmetic(new SimpleExpression(), Operator.ADD, - new SetValueExpression(new LongFieldValue(69))).execute()); - assertNull(newArithmetic(new SetValueExpression(new LongFieldValue(69)), Operator.ADD, + new ConstantExpression(new LongFieldValue(69))).execute()); + assertNull(newArithmetic(new ConstantExpression(new LongFieldValue(69)), Operator.ADD, new SimpleExpression()).execute()); } @Test public void requireThatNonNumericOperandThrows() { try { - newArithmetic(new SetValueExpression(new IntegerFieldValue(6)), Operator.ADD, - new SetValueExpression(new StringFieldValue("9"))).execute(); + newArithmetic(new ConstantExpression(new IntegerFieldValue(6)), Operator.ADD, + new ConstantExpression(new StringFieldValue("9"))).execute(); fail(); } catch (IllegalArgumentException e) { assertEquals("Unsupported operation: [int] + [string]", e.getMessage()); } try { - newArithmetic(new SetValueExpression(new StringFieldValue("6")), Operator.ADD, - new SetValueExpression(new IntegerFieldValue(9))).execute(); + newArithmetic(new ConstantExpression(new StringFieldValue("6")), Operator.ADD, + new ConstantExpression(new IntegerFieldValue(9))).execute(); fail(); } catch (IllegalArgumentException e) { assertEquals("Unsupported operation: [string] + [int]", e.getMessage()); @@ -156,8 +156,8 @@ public class ArithmeticTestCase { } private void assertResult(FieldValue lhs, Operator op, FieldValue rhs, FieldValue expected) { - assertEquals(expected, evaluate(new SetValueExpression(lhs), op, - new SetValueExpression(rhs))); + assertEquals(expected, evaluate(new ConstantExpression(lhs), op, + new ConstantExpression(rhs))); } private void assertType(DataType lhs, Operator op, DataType rhs, DataType expected) { @@ -178,15 +178,15 @@ public class ArithmeticTestCase { } private static ArithmeticExpression newArithmetic(FieldValue lhs, Operator op, FieldValue rhs) { - return newArithmetic(new SetValueExpression(lhs), op, new SetValueExpression(rhs)); + return newArithmetic(new ConstantExpression(lhs), op, new ConstantExpression(rhs)); } private static ArithmeticExpression newArithmetic(Expression lhs, Operator op, Expression rhs) { return new ArithmeticExpression(lhs, op, rhs); } - private static SetValueExpression newLong(long val) { - return new SetValueExpression(new LongFieldValue(val)); + private static ConstantExpression newLong(long val) { + return new ConstantExpression(new LongFieldValue(val)); } private static void assertVerify(Expression lhs, Operator op, Expression rhs, DataType val) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java index fee467b4c2c..eec95aa6644 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java @@ -48,7 +48,7 @@ public class Base64DecodeTestCase { new Base64DecodeExpression().execute(new StringFieldValue("abcdefghijlkm")); fail(); } catch (IllegalArgumentException e) { - assertEquals("Base64 value 'abcdefghijlkm' is out of range.", e.getMessage()); + assertEquals("Base64 value 'abcdefghijlkm' is out of range", e.getMessage()); } } @@ -66,7 +66,7 @@ public class Base64DecodeTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new Base64DecodeExpression(); assertVerify(DataType.STRING, exp, DataType.LONG); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.LONG, exp, "Expected string input, got long."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.LONG, exp, "Expected string input, got long"); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java index 50c8d540e81..d82044c6c80 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java @@ -40,7 +40,7 @@ public class Base64EncodeTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new Base64EncodeExpression(); assertVerify(DataType.LONG, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected long input, got null."); - assertVerifyThrows(DataType.STRING, exp, "Expected long input, got string."); + assertVerifyThrows(null, exp, "Expected long input, but no input is specified"); + assertVerifyThrows(DataType.STRING, exp, "Expected long input, got string"); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java index 09341a6f9ba..d4f97c62e7d 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java @@ -44,22 +44,22 @@ public class CatTestCase { @Test public void requireThatExpressionCanBeVerified() { - assertVerify(new SetValueExpression(new StringFieldValue("foo")), - new SetValueExpression(new StringFieldValue("bar")), null); + assertVerify(new ConstantExpression(new StringFieldValue("foo")), + new ConstantExpression(new StringFieldValue("bar")), null); assertVerify(new SimpleExpression(DataType.STRING), new SimpleExpression(DataType.STRING), DataType.STRING); assertVerifyThrows(new SimpleExpression().setCreatedOutput(null), new SimpleExpression().setCreatedOutput(DataType.STRING), null, - "Attempting to concatenate a null value "); + "Attempting to concatenate a null value"); assertVerifyThrows(new SimpleExpression(DataType.STRING), new SimpleExpression(DataType.INT), null, - "Operands require conflicting input types, string vs int."); + "Operands require conflicting input types, string vs int"); assertVerifyThrows(new SimpleExpression(DataType.STRING), new SimpleExpression(DataType.STRING), null, - "Expected string input, got null."); + "Expected string input, but no input is specified"); assertVerifyThrows(new SimpleExpression(DataType.STRING), new SimpleExpression(DataType.STRING), DataType.INT, - "Expected string input, got int."); + "Expected string input, got int"); } @Test @@ -89,16 +89,16 @@ public class CatTestCase { @Test public void requireThatInputValueIsAvailableToAllInnerExpressions() { assertEquals(new StringFieldValue("foobarfoo"), - new StatementExpression(new SetValueExpression(new StringFieldValue("foo")), + new StatementExpression(new ConstantExpression(new StringFieldValue("foo")), new CatExpression(new ThisExpression(), - new SetValueExpression(new StringFieldValue("bar")), + new ConstantExpression(new StringFieldValue("bar")), new ThisExpression())).execute()); } @Test public void requiredThatRequiredInputTypeAllowsNull() { - assertVerify(new SetValueExpression(new StringFieldValue("foo")), new TrimExpression(), DataType.STRING); - assertVerify(new TrimExpression(), new SetValueExpression(new StringFieldValue("foo")), DataType.STRING); + assertVerify(new ConstantExpression(new StringFieldValue("foo")), new TrimExpression(), DataType.STRING); + assertVerify(new TrimExpression(), new ConstantExpression(new StringFieldValue("foo")), DataType.STRING); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpressionTestCase.java index 6ad01e1e9bf..3d2230bb524 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpressionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpressionTestCase.java @@ -15,7 +15,7 @@ public class CompositeExpressionTestCase { @Test public void requireThatToScriptBlockOutputIsParsable() throws ParseException { - Expression exp = new SetValueExpression(new IntegerFieldValue(69)); + Expression exp = new ConstantExpression(new IntegerFieldValue(69)); assertScript("{ 69; }", exp); assertScript("{ 69; }", new StatementExpression(exp)); assertScript("{ 69; }", new ScriptExpression(new StatementExpression(exp))); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java index 29a0cd1617d..4dd5ef4d9f9 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java @@ -53,6 +53,6 @@ public class EchoTestCase { Expression exp = new EchoExpression(); assertVerify(DataType.INT, exp, DataType.INT); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java index e993f6cbdee..d1d62442857 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java @@ -75,8 +75,8 @@ public class ExactTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new ExactExpression(); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); } private static void assertAnnotation(int expectedFrom, int expectedLen, StringFieldValue expectedVal, diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionTestCase.java index d3ff552ab10..e42ab5a60d5 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionTestCase.java @@ -34,22 +34,22 @@ public class ExpressionTestCase { public void requireThatInputTypeIsCheckedBeforeVerify() { assertVerify(newRequiredInput(DataType.INT), DataType.INT); assertVerifyThrows(newRequiredInput(DataType.INT), null, - "Expected int input, got null."); + "Expected int input, but no input is specified"); assertVerifyThrows(newRequiredInput(DataType.INT), UnresolvedDataType.INSTANCE, - "Failed to resolve input type."); + "Failed to resolve input type"); assertVerifyThrows(newRequiredInput(DataType.INT), DataType.STRING, - "Expected int input, got string."); + "Expected int input, got string"); } @Test public void requireThatOutputTypeIsCheckedAfterVerify() { assertVerify(newCreatedOutput(DataType.INT, DataType.INT), null); assertVerifyThrows(newCreatedOutput(DataType.INT, (DataType)null), null, - "Expected int output, got null."); + "Expected int output, but no output is specified"); assertVerifyThrows(newCreatedOutput(DataType.INT, UnresolvedDataType.INSTANCE), null, - "Failed to resolve output type."); + "Failed to resolve output type"); assertVerifyThrows(newCreatedOutput(DataType.INT, DataType.STRING), null, - "Expected int output, got string."); + "Expected int output, got string"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenTestCase.java index ec0af3ba5f2..b875abb923b 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenTestCase.java @@ -87,7 +87,7 @@ public class FlattenTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new FlattenExpression(); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java index 356910a299a..04a682aa82a 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java @@ -53,9 +53,9 @@ public class ForEachTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new ForEachExpression(SimpleExpression.newConversion(DataType.INT, DataType.STRING)); assertVerify(DataType.getArray(DataType.INT), exp, DataType.getArray(DataType.STRING)); - assertVerifyThrows(null, exp, "Expected any input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected Array, Struct or WeightedSet input, got int."); - assertVerifyThrows(DataType.getArray(DataType.STRING), exp, "Expected int input, got string."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected Array, Struct or WeightedSet input, got int"); + assertVerifyThrows(DataType.getArray(DataType.STRING), exp, "Expected int input, got string"); } @Test @@ -64,9 +64,9 @@ public class ForEachTestCase { type.addField(new Field("foo", DataType.INT)); assertVerify(type, new ForEachExpression(new SimpleExpression()), type); assertVerifyThrows(type, new ForEachExpression(SimpleExpression.newConversion(DataType.STRING, DataType.INT)), - "Expected string input, got int."); + "Expected string input, got int"); assertVerifyThrows(type, new ForEachExpression(SimpleExpression.newConversion(DataType.INT, DataType.STRING)), - "Expected int output, got string."); + "Expected int output, got string"); } @Test @@ -94,7 +94,7 @@ public class ForEachTestCase { @Test public void requireThatCreatedOutputTypeDependsOnInnerExpression() { assertNull(new ForEachExpression(new SimpleExpression()).createdOutputType()); - assertNotNull(new ForEachExpression(new SetValueExpression(new IntegerFieldValue(69))).createdOutputType()); + assertNotNull(new ForEachExpression(new ConstantExpression(new IntegerFieldValue(69))).createdOutputType()); } @Test @@ -134,7 +134,7 @@ public class ForEachTestCase { new ForEachExpression(new SimpleExpression()).execute(new StringFieldValue("foo")); fail(); } catch (IllegalArgumentException e) { - assertEquals("Expected Array, Struct or WeightedSet input, got string.", e.getMessage()); + assertEquals("Expected Array, Struct or WeightedSet input, got string", e.getMessage()); } } @@ -215,7 +215,7 @@ public class ForEachTestCase { new ForEachExpression(new ToArrayExpression()).verify(ctx); fail(); } catch (VerificationException e) { - assertEquals("Expected int output, got Array<int>.", e.getMessage()); + assertEquals("Expected int output, got Array<int>", e.getMessage()); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldTestCase.java index 9fbb95c0def..5da7a3544dc 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldTestCase.java @@ -38,8 +38,8 @@ public class GetFieldTestCase { type.addField(new Field("foo", DataType.STRING)); Expression exp = new GetFieldExpression("foo"); assertVerify(type, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected any input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected structured input, got int."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected structured input, got int"); assertVerifyThrows(type, new GetFieldExpression("bar"), "Field 'bar' not found in struct type 'my_struct'"); } @@ -69,7 +69,7 @@ public class GetFieldTestCase { new GetFieldExpression("foo").execute(new StringFieldValue("bar")); fail(); } catch (IllegalArgumentException e) { - assertEquals("Expected structured input, got string.", e.getMessage()); + assertEquals("Expected structured input, got string", e.getMessage()); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java index 30ee50fccc8..daa02edcaa2 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java @@ -41,7 +41,7 @@ public class GetVarTestCase { new GetVarExpression("bar").verify(ctx); fail(); } catch (VerificationException e) { - assertEquals("Variable 'bar' not found.", e.getMessage()); + assertEquals("Variable 'bar' not found", e.getMessage()); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java index 092ca6b5f59..b0fa561a62e 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java @@ -46,8 +46,8 @@ public class GuardTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new GuardExpression(SimpleExpression.newConversion(DataType.INT, DataType.STRING)); assertVerify(DataType.INT, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected int input, got null."); - assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string."); + assertVerifyThrows(null, exp, "Expected int input, but no input is specified"); + assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java index b92d7988df2..c42cfec5847 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java @@ -29,8 +29,8 @@ public class HexDecodeTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new HexDecodeExpression(); assertVerify(DataType.STRING, exp, DataType.LONG); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.LONG, exp, "Expected string input, got long."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.LONG, exp, "Expected string input, got long"); } @Test @@ -67,7 +67,7 @@ public class HexDecodeTestCase { new HexDecodeExpression().execute(new StringFieldValue("1ffffffffffffffff")); fail(); } catch (IllegalArgumentException e) { - assertEquals("Hex value '1ffffffffffffffff' is out of range.", e.getMessage()); + assertEquals("Hex value '1ffffffffffffffff' is out of range", e.getMessage()); } } @@ -77,7 +77,7 @@ public class HexDecodeTestCase { new HexDecodeExpression().execute(new StringFieldValue("???")); fail(); } catch (IllegalArgumentException e) { - assertEquals("Illegal hex value '???'.", e.getMessage()); + assertEquals("Illegal hex value '???'", e.getMessage()); } } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java index 14745e5d61d..23c55327ed6 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java @@ -29,8 +29,8 @@ public class HexEncodeTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new HexEncodeExpression(); assertVerify(DataType.LONG, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected long input, got null."); - assertVerifyThrows(DataType.STRING, exp, "Expected long input, got string."); + assertVerifyThrows(null, exp, "Expected long input, but no input is specified"); + assertVerifyThrows(DataType.STRING, exp, "Expected long input, got string"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java index d429d340480..6c8980c8c1d 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java @@ -42,29 +42,29 @@ public class IfThenTestCase { Expression exp = newRequiredInput(DataType.STRING, Comparator.EQ, DataType.STRING, DataType.STRING, DataType.STRING); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); assertVerifyThrows(null, () -> newRequiredInput(DataType.INT, Comparator.EQ, DataType.STRING, DataType.STRING, DataType.STRING), - "Operands require conflicting input types, int vs string."); + "Operands require conflicting input types, int vs string"); assertVerifyThrows(null, () -> newRequiredInput(DataType.STRING, Comparator.EQ, DataType.INT, DataType.STRING, DataType.STRING), - "Operands require conflicting input types, string vs int."); + "Operands require conflicting input types, string vs int"); assertVerifyThrows(null, () -> newRequiredInput(DataType.STRING, Comparator.EQ, DataType.STRING, DataType.INT, DataType.STRING), - "Operands require conflicting input types, string vs int."); + "Operands require conflicting input types, string vs int"); assertVerifyThrows(null, () -> newRequiredInput(DataType.STRING, Comparator.EQ, DataType.STRING, DataType.STRING, DataType.INT), - "Operands require conflicting input types, string vs int."); + "Operands require conflicting input types, string vs int"); } @Test public void requireThatExpressionCanBeVerified() { assertVerify(DataType.STRING, new FlattenExpression(), DataType.STRING); assertVerifyThrows(null, new FlattenExpression(), - "Expected string input, got null."); + "Expected string input, but no input is specified"); assertVerifyThrows(DataType.INT, new FlattenExpression(), - "Expected string input, got int."); + "Expected string input, got int"); } @Test @@ -116,7 +116,7 @@ public class IfThenTestCase { @Test public void requireThatAllChildrenSeeInputValue() { FieldValueAdapter adapter = createTestAdapter(); - new StatementExpression(new SetValueExpression(new IntegerFieldValue(69)), + new StatementExpression(new ConstantExpression(new IntegerFieldValue(69)), new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ, new AttributeExpression("rhs"), @@ -128,7 +128,7 @@ public class IfThenTestCase { assertNull(null, adapter.getInputValue("ifFalse")); adapter = createTestAdapter(); - new StatementExpression(new SetValueExpression(new IntegerFieldValue(69)), + new StatementExpression(new ConstantExpression(new IntegerFieldValue(69)), new IfThenExpression(new AttributeExpression("lhs"), Comparator.NE, new AttributeExpression("rhs"), @@ -143,10 +143,10 @@ public class IfThenTestCase { @Test public void requireThatElseExpIsOptional() { ExecutionContext ctx = new ExecutionContext(); - Expression exp = new IfThenExpression(new SetValueExpression(new IntegerFieldValue(6)), + Expression exp = new IfThenExpression(new ConstantExpression(new IntegerFieldValue(6)), Comparator.GT, - new SetValueExpression(new IntegerFieldValue(9)), - new SetValueExpression(new StringFieldValue("69"))); + new ConstantExpression(new IntegerFieldValue(9)), + new ConstantExpression(new StringFieldValue("69"))); FieldValue val = ctx.setValue(new IntegerFieldValue(96)).execute(exp).getValue(); assertTrue(val instanceof IntegerFieldValue); assertEquals(96, ((IntegerFieldValue)val).getInteger()); @@ -225,8 +225,8 @@ public class IfThenTestCase { @Test public void requireThatNullLeftOrRightHandSideEvaluatesToNull() { Expression exp = new IfThenExpression(new GetVarExpression("lhs"), Comparator.EQ, new GetVarExpression("rhs"), - new SetValueExpression(new StringFieldValue("true")), - new SetValueExpression(new StringFieldValue("false"))); + new ConstantExpression(new StringFieldValue("true")), + new ConstantExpression(new StringFieldValue("false"))); assertEquals(new StringFieldValue("true"), exp.execute(new ExecutionContext().setVariable("lhs", new IntegerFieldValue(69)) .setVariable("rhs", new IntegerFieldValue(69)))); @@ -237,6 +237,25 @@ public class IfThenTestCase { assertNull(exp.execute(new ExecutionContext().setVariable("rhs", new IntegerFieldValue(69)))); } + @Test + public void testRequiredInputType() { + var ifExpression = new IfThenExpression(new InputExpression("field1"), + Comparator.EQ, + new ConstantExpression(new IntegerFieldValue(0)), + wrapLikeTheParser(new ConstantExpression(new StringFieldValue("true"))), + wrapLikeTheParser(new ConstantExpression(new StringFieldValue("false")))); + assertNull(ifExpression.requiredInputType()); + assertEquals(DataType.STRING, ifExpression.createdOutputType()); + + var expression = new ScriptExpression(new StatementExpression(ifExpression, + new AttributeExpression(null))); + assertNull(expression.requiredInputType()); + } + + private Expression wrapLikeTheParser(Expression expression) { + return new ScriptExpression(new StatementExpression(expression)); + } + private static void assertCmpTrue(FieldValue lhs, Comparator cmp, FieldValue rhs) { assertTrue(evaluateIfThen(lhs, cmp, rhs)); } @@ -248,8 +267,8 @@ public class IfThenTestCase { private static boolean evaluateIfThen(FieldValue lhs, Comparator cmp, FieldValue rhs) { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); new StatementExpression( - new SetValueExpression(new IntegerFieldValue(1)), - new IfThenExpression(new SetValueExpression(lhs), cmp, new SetValueExpression(rhs), + new ConstantExpression(new IntegerFieldValue(1)), + new IfThenExpression(new ConstantExpression(lhs), cmp, new ConstantExpression(rhs), new SetVarExpression("true"), new SetVarExpression("false"))).execute(ctx); return ctx.getVariable("true") != null; diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java index fe16e8d10ab..0cfbb23bd1b 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java @@ -40,7 +40,7 @@ public class InputTestCase { new InputExpression("bar").verify(adapter); fail(); } catch (VerificationException e) { - assertEquals("Field 'bar' not found.", e.getMessage()); + assertEquals("Field 'bar' not found", e.getMessage()); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java index a1eb46afd33..b4c94166c2d 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java @@ -36,8 +36,8 @@ public class JoinTestCase { Expression exp = new JoinExpression(";"); assertVerify(DataType.getArray(DataType.INT), exp, DataType.STRING); assertVerify(DataType.getArray(DataType.STRING), exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected any input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected Array input, got int."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected Array input, got int"); } @Test @@ -58,7 +58,7 @@ public class JoinTestCase { new JoinExpression(";").execute(new StringFieldValue("foo")); fail(); } catch (IllegalArgumentException e) { - assertEquals("Expected Array input, got string.", e.getMessage()); + assertEquals("Expected Array input, got string", e.getMessage()); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java index ce2251d1884..f1d68f6438a 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java @@ -28,8 +28,8 @@ public class LowerCaseTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new LowerCaseExpression(); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java index 1f9afdea44d..0bcbcb2b1b3 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java @@ -93,7 +93,7 @@ public class MathResolverTestCase { // -------------------------------------------------------------------------------- private static Expression newInteger(int val) { - return new SetValueExpression(new IntegerFieldValue(val)); + return new ConstantExpression(new IntegerFieldValue(val)); } private static int evaluate(Expression exp) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java index 4d4f11416cb..ae52ad83e8c 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java @@ -46,8 +46,8 @@ public class NGramTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new NGramExpression(new SimpleLinguistics(), 69); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java index f00ad6a95aa..9584bfa1438 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java @@ -43,8 +43,8 @@ public class NormalizeTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new NormalizeExpression(new SimpleLinguistics()); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java index 28dc12781f0..685416c905a 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java @@ -76,21 +76,21 @@ public class OptimizePredicateTestCase { @Test public void requireThatExpressionCanBeVerified() { Expression exp = new OptimizePredicateExpression(); - assertVerifyThrows(null, exp, "Expected predicate input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected predicate input, got int."); - assertVerifyThrows(DataType.PREDICATE, exp, "Variable 'arity' must be set."); + assertVerifyThrows(null, exp, "Expected predicate input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected predicate input, got int"); + assertVerifyThrows(DataType.PREDICATE, exp, "Variable 'arity' must be set"); VerificationContext context = new VerificationContext().setValueType(DataType.PREDICATE); context.setVariable("arity", DataType.STRING); - assertVerifyCtxThrows(context, exp, "Variable 'arity' must have type int."); + assertVerifyCtxThrows(context, exp, "Variable 'arity' must have type int"); context.setVariable("arity", DataType.INT); assertVerifyCtx(context, exp, DataType.PREDICATE); context.setVariable("lower_bound", DataType.INT); - assertVerifyCtxThrows(context, exp, "Variable 'lower_bound' must have type long."); + assertVerifyCtxThrows(context, exp, "Variable 'lower_bound' must have type long"); context.setVariable("lower_bound", DataType.LONG); assertVerifyCtx(context, exp, DataType.PREDICATE); context.setVariable("upper_bound", DataType.INT); - assertVerifyCtxThrows(context, exp, "Variable 'upper_bound' must have type long."); + assertVerifyCtxThrows(context, exp, "Variable 'upper_bound' must have type long"); context.setVariable("upper_bound", DataType.LONG); assertVerifyCtx(context, exp, DataType.PREDICATE); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java index 26bc531f806..10dab2061e7 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java @@ -27,7 +27,7 @@ class OutputAssert { public static void assertVerify(OutputExpression exp) { assertVerify(new MyAdapter(null), DataType.INT, exp); assertVerify(new MyAdapter(null), DataType.STRING, exp); - assertVerifyThrows(new MyAdapter(null), null, exp, "Expected any input, got null."); + assertVerifyThrows(new MyAdapter(null), null, exp, "Expected any input, but no input is specified"); assertVerifyThrows(new MyAdapter(new VerificationException((Expression) null, "foo")), DataType.INT, exp, "foo"); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java index 9f3a220249b..5ca3e336f69 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java @@ -37,8 +37,8 @@ public class ParenthesisTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new ParenthesisExpression(SimpleExpression.newConversion(DataType.INT, DataType.STRING)); assertVerify(DataType.INT, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected int input, got null."); - assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string."); + assertVerifyThrows(null, exp, "Expected int input, but no input is specified"); + assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java index c648bb12ede..62b960568e9 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java @@ -54,20 +54,20 @@ public class ScriptTestCase { @Test public void requireThatExpressionCanBeVerified() { Expression exp = newScript(newStatement(SimpleExpression.newConversion(DataType.INT, DataType.STRING))); - assertVerify(DataType.INT, exp, DataType.INT); // does not touch output - assertVerifyThrows(null, exp, "Expected int input, got null."); - assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string."); + assertVerify(DataType.INT, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected int input, but no input is specified"); + assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string"); assertVerifyThrows(null, () -> newScript(newStatement(SimpleExpression.newConversion(DataType.INT, DataType.STRING)), newStatement(SimpleExpression.newConversion(DataType.STRING, DataType.INT))), - "Statements require conflicting input types, int vs string."); + "Statements require conflicting input types, int vs string"); } @Test public void requireThatInputValueIsAvailableToAllStatements() { SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out-1", DataType.INT), new Field("out-2", DataType.INT)); - newStatement(new SetValueExpression(new IntegerFieldValue(69)), + newStatement(new ConstantExpression(new IntegerFieldValue(69)), newScript(newStatement(new AttributeExpression("out-1"), new AttributeExpression("out-2")))).execute(adapter); assertEquals(new IntegerFieldValue(69), adapter.getInputValue("out-1")); @@ -90,8 +90,8 @@ public class ScriptTestCase { @Test public void requireThatScriptEvaluatesToInputValue() { SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out", DataType.INT)); - newStatement(new SetValueExpression(new IntegerFieldValue(6)), - newScript(newStatement(new SetValueExpression(new IntegerFieldValue(9)))), + newStatement(new ConstantExpression(new IntegerFieldValue(6)), + newScript(newStatement(new ConstantExpression(new IntegerFieldValue(9)))), new AttributeExpression("out")).execute(adapter); assertEquals(new IntegerFieldValue(6), adapter.getInputValue("out")); } @@ -99,7 +99,7 @@ public class ScriptTestCase { @Test public void requireThatVariablesAreAvailableInScript() { SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out", DataType.INT)); - newScript(newStatement(new SetValueExpression(new IntegerFieldValue(69)), + newScript(newStatement(new ConstantExpression(new IntegerFieldValue(69)), new SetVarExpression("tmp")), newStatement(new GetVarExpression("tmp"), new AttributeExpression("out"))).execute(adapter); @@ -109,7 +109,7 @@ public class ScriptTestCase { @Test public void requireThatVariablesAreAvailableOutsideScript() { SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out", DataType.INT)); - newStatement(newScript(newStatement(new SetValueExpression(new IntegerFieldValue(69)), + newStatement(newScript(newStatement(new ConstantExpression(new IntegerFieldValue(69)), new SetVarExpression("tmp"))), new GetVarExpression("tmp"), new AttributeExpression("out")).execute(adapter); @@ -119,9 +119,9 @@ public class ScriptTestCase { @Test public void requireThatVariablesReplaceOthersOutsideScript() { SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out", DataType.INT)); - newStatement(new SetValueExpression(new IntegerFieldValue(6)), + newStatement(new ConstantExpression(new IntegerFieldValue(6)), new SetVarExpression("tmp"), - newScript(newStatement(new SetValueExpression(new IntegerFieldValue(9)), + newScript(newStatement(new ConstantExpression(new IntegerFieldValue(9)), new SetVarExpression("tmp"))), new GetVarExpression("tmp"), new AttributeExpression("out")).execute(adapter); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java index 46605cc5f6a..f4d26bee9f1 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java @@ -61,16 +61,16 @@ public class SelectInputTestCase { assertVerifyThrows(adapter, newSelectInput(new AttributeExpression("my_int"), "my_str"), "Can not assign string to field 'my_int' which is int."); assertVerifyThrows(adapter, newSelectInput(new AttributeExpression("my_int"), "my_unknown"), - "Field 'my_unknown' not found."); + "Field 'my_unknown' not found"); } @Test public void requireThatSelectedExpressionIsRun() { - assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("foo"), "foo"); - assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("bar"), "bar"); - assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("foo", "bar"), "foo"); - assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("bar", "baz"), "bar"); - assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("baz", "cox"), null); + assertSelect(Arrays.asList("foo", "bar"), List.of("foo"), "foo"); + assertSelect(Arrays.asList("foo", "bar"), List.of("bar"), "bar"); + assertSelect(Arrays.asList("foo", "bar"), List.of("foo", "bar"), "foo"); + assertSelect(Arrays.asList("foo", "bar"), List.of("bar", "baz"), "bar"); + assertSelect(Arrays.asList("foo", "bar"), List.of("baz", "cox"), null); } private static void assertVerify(FieldTypeAdapter adapter, DataType value, Expression exp) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java index 9b95ffc31bb..0821ab0cc40 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java @@ -29,8 +29,8 @@ public class SetLanguageTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new SetLanguageExpression(); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java index cace2c5923d..488a8c098fd 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java @@ -19,23 +19,23 @@ public class SetValueTestCase { @Test public void requireThatAccessorsWork() { FieldValue foo = new StringFieldValue("foo"); - SetValueExpression exp = new SetValueExpression(foo); + ConstantExpression exp = new ConstantExpression(foo); assertSame(foo, exp.getValue()); } @Test public void requireThatHashCodeAndEqualsAreImplemented() { FieldValue foo = new StringFieldValue("foo"); - Expression exp = new SetValueExpression(foo); + Expression exp = new ConstantExpression(foo); assertFalse(exp.equals(new Object())); - assertFalse(exp.equals(new SetValueExpression(new StringFieldValue("bar")))); - assertEquals(exp, new SetValueExpression(foo)); - assertEquals(exp.hashCode(), new SetValueExpression(foo).hashCode()); + assertFalse(exp.equals(new ConstantExpression(new StringFieldValue("bar")))); + assertEquals(exp, new ConstantExpression(foo)); + assertEquals(exp.hashCode(), new ConstantExpression(foo).hashCode()); } @Test public void requireThatExpressionCanBeVerified() { - Expression exp = new SetValueExpression(new StringFieldValue("foo")); + Expression exp = new ConstantExpression(new StringFieldValue("foo")); assertVerify(null, exp, DataType.STRING); assertVerify(DataType.INT, exp, DataType.STRING); assertVerify(DataType.STRING, exp, DataType.STRING); @@ -44,7 +44,7 @@ public class SetValueTestCase { @Test public void requireThatNullValueThrowsException() { try { - new SetValueExpression(null); + new ConstantExpression(null); fail(); } catch (NullPointerException e) { @@ -54,12 +54,12 @@ public class SetValueTestCase { @Test public void requireThatValueIsSet() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - new SetValueExpression(new StringFieldValue("69")).execute(ctx); + new ConstantExpression(new StringFieldValue("69")).execute(ctx); assertEquals(new StringFieldValue("69"), ctx.getValue()); } @Test public void requireThatLongFieldValueGetsATrailingL() { - assertEquals("69L", new SetValueExpression(new LongFieldValue(69)).toString()); + assertEquals("69L", new ConstantExpression(new LongFieldValue(69)).toString()); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java index 0cae20af29d..ae393837dfe 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java @@ -36,13 +36,13 @@ public class SetVarTestCase { Expression exp = new SetVarExpression("foo"); assertVerify(DataType.INT, exp, DataType.INT); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); try { new VerificationContext().setVariable("foo", DataType.INT).setValueType(DataType.STRING).execute(exp); fail(); } catch (VerificationException e) { - assertEquals("Attempting to assign conflicting types to variable 'foo', int vs string.", e.getMessage()); + assertEquals("Attempting to assign conflicting types to variable 'foo', int vs string", e.getMessage()); } } @@ -69,7 +69,7 @@ public class SetVarTestCase { fail(); } catch (VerificationException e) { assertTrue(e.getExpressionType().equals(SetVarExpression.class)); - assertEquals("Attempting to assign conflicting types to variable 'out', int vs string.", e.getMessage()); + assertEquals("Attempting to assign conflicting types to variable 'out', int vs string", e.getMessage()); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java index db51158d5c2..80ca6622001 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java @@ -64,29 +64,14 @@ final class SimpleExpression extends Expression { } @Override - public boolean equals(Object obj) { - if (!(obj instanceof SimpleExpression)) { - return false; - } - SimpleExpression rhs = (SimpleExpression)obj; - if (hasExecuteValue != rhs.hasExecuteValue) { - return false; - } - if (!equals(executeValue, rhs.executeValue)) { - return false; - } - if (hasVerifyValue != rhs.hasVerifyValue) { - return false; - } - if (!equals(verifyValue, rhs.verifyValue)) { - return false; - } - if (!equals(requiredInputType(), rhs.requiredInputType())) { - return false; - } - if (!equals(createdOutput, rhs.createdOutput)) { - return false; - } + public boolean equals(Object o) { + if (!(o instanceof SimpleExpression other)) return false; + if (hasExecuteValue != other.hasExecuteValue) return false; + if (!equals(executeValue, other.executeValue)) return false; + if (hasVerifyValue != other.hasVerifyValue) return false; + if (!equals(verifyValue, other.verifyValue)) return false; + if (!equals(requiredInputType(), other.requiredInputType())) return false; + if (!equals(createdOutput, other.createdOutput)) return false; return true; } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java index 7ad4db41e33..d9a8dfcebe6 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java @@ -37,8 +37,8 @@ public class SplitTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new SplitExpression(";"); assertVerify(DataType.STRING, exp, DataType.getArray(DataType.STRING)); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java index 1a1442f68ac..6ceadff8b28 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java @@ -90,9 +90,9 @@ public class StatementTestCase { public void requireThatInternalVerificationIsPerformed() { Expression exp = newStatement(SimpleExpression.newOutput(DataType.STRING), SimpleExpression.newConversion(DataType.INT, DataType.STRING)); - assertVerifyThrows(null, exp, "Expected int input, got string."); - assertVerifyThrows(DataType.INT, exp, "Expected int input, got string."); - assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string."); + assertVerifyThrows(null, exp, "Expected int input, got string"); + assertVerifyThrows(DataType.INT, exp, "Expected int input, got string"); + assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string"); exp = newStatement(SimpleExpression.newOutput(DataType.INT), SimpleExpression.newConversion(DataType.INT, DataType.STRING)); @@ -104,7 +104,7 @@ public class StatementTestCase { @Test public void requireThatStatementIsExecuted() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - StatementExpression statement = newStatement(new SetValueExpression(new IntegerFieldValue(69))); + StatementExpression statement = newStatement(new ConstantExpression(new IntegerFieldValue(69))); newStatement(statement).execute(ctx); FieldValue val = ctx.getValue(); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java index f6bed7831ae..b209533f379 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java @@ -37,8 +37,8 @@ public class SubstringTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new SubstringExpression(6, 9); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java index c79d38f5650..5d8be76c1b4 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java @@ -58,8 +58,8 @@ public class SwitchTestCase { Expression foo = SimpleExpression.newConversion(DataType.STRING, DataType.INT); Expression exp = new SwitchExpression(Collections.singletonMap("foo", foo)); assertVerify(DataType.STRING, exp, DataType.STRING); // does not touch output - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); } @Test @@ -67,10 +67,10 @@ public class SwitchTestCase { Map<String, Expression> cases = new HashMap<>(); cases.put("foo", SimpleExpression.newRequired(DataType.INT)); assertVerifyThrows(DataType.STRING, new SwitchExpression(cases), - "Expected int input, got string."); + "Expected int input, got string"); assertVerifyThrows(DataType.STRING, new SwitchExpression(Collections.<String, Expression>emptyMap(), SimpleExpression.newRequired(DataType.INT)), - "Expected int input, got string."); + "Expected int input, got string"); } @Test @@ -79,7 +79,7 @@ public class SwitchTestCase { new SwitchExpression(Collections.<String, Expression>emptyMap()).execute(new IntegerFieldValue(69)); fail(); } catch (IllegalArgumentException e) { - assertEquals("Expected string input, got int.", e.getMessage()); + assertEquals("Expected string input, got int", e.getMessage()); } } @@ -104,9 +104,9 @@ public class SwitchTestCase { @Test public void requireThatCorrectExpressionIsExecuted() { Map<String, Expression> cases = new HashMap<>(); - cases.put("foo", new StatementExpression(new SetValueExpression(new StringFieldValue("bar")), + cases.put("foo", new StatementExpression(new ConstantExpression(new StringFieldValue("bar")), new SetVarExpression("out"))); - cases.put("baz", new StatementExpression(new SetValueExpression(new StringFieldValue("cox")), + cases.put("baz", new StatementExpression(new ConstantExpression(new StringFieldValue("cox")), new SetVarExpression("out"))); Expression exp = new SwitchExpression(cases); assertEvaluate(new StringFieldValue("foo"), exp, new StringFieldValue("bar")); @@ -117,9 +117,9 @@ public class SwitchTestCase { @Test public void requireThatDefaultExpressionIsExecuted() { Map<String, Expression> cases = new HashMap<>(); - cases.put("foo", new StatementExpression(new SetValueExpression(new StringFieldValue("bar")), + cases.put("foo", new StatementExpression(new ConstantExpression(new StringFieldValue("bar")), new SetVarExpression("out"))); - Expression defaultExp = new StatementExpression(new SetValueExpression(new StringFieldValue("cox")), + Expression defaultExp = new StatementExpression(new ConstantExpression(new StringFieldValue("cox")), new SetVarExpression("out")); Expression exp = new SwitchExpression(cases, defaultExp); assertEvaluate(new StringFieldValue("foo"), exp, new StringFieldValue("bar")); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java index e33c8330d8e..7fc1f2fb4bd 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java @@ -29,7 +29,7 @@ public class ThisTestCase { Expression exp = new ThisExpression(); assertVerify(DataType.INT, exp, DataType.INT); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java index 294f6f2e266..e7a7ef5aee5 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java @@ -33,7 +33,7 @@ public class ToArrayTestCase { Expression exp = new ToArrayExpression(); assertVerify(DataType.INT, exp, DataType.getArray(DataType.INT)); assertVerify(DataType.STRING, exp, DataType.getArray(DataType.STRING)); - assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToBoolTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToBoolTestCase.java index 920f186c468..cac754e2b77 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToBoolTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToBoolTestCase.java @@ -34,7 +34,7 @@ public class ToBoolTestCase { Expression exp = new ToBoolExpression(); assertVerify(DataType.INT, exp, DataType.BOOL); assertVerify(DataType.STRING, exp, DataType.BOOL); - assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java index 76e075b4315..4bee5953b8c 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java @@ -30,7 +30,7 @@ public class ToByteTestCase { Expression exp = new ToByteExpression(); assertVerify(DataType.INT, exp, DataType.BYTE); assertVerify(DataType.STRING, exp, DataType.BYTE); - assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java index ab0263ee134..39f347417b1 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java @@ -30,7 +30,7 @@ public class ToDoubleTestCase { Expression exp = new ToDoubleExpression(); assertVerify(DataType.INT, exp, DataType.DOUBLE); assertVerify(DataType.STRING, exp, DataType.DOUBLE); - assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpressionTestCase.java index 7203afcc1a0..3a6bf85f972 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpressionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpressionTestCase.java @@ -27,8 +27,8 @@ public class ToEpochSecondExpressionTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new ToEpochSecondExpression(); assertVerify(DataType.STRING, exp, DataType.LONG); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); - assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java index d36b7c88da3..14b0bf57a45 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java @@ -30,7 +30,7 @@ public class ToFloatTestCase { Expression exp = new ToFloatExpression(); assertVerify(DataType.INT, exp, DataType.FLOAT); assertVerify(DataType.STRING, exp, DataType.FLOAT); - assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java index 7831c6f675f..56c26cba19b 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java @@ -30,7 +30,7 @@ public class ToIntegerTestCase { Expression exp = new ToIntegerExpression(); assertVerify(DataType.INT, exp, DataType.INT); assertVerify(DataType.STRING, exp, DataType.INT); - assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java index f586b8ce110..60f7a84f044 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java @@ -30,7 +30,7 @@ public class ToLongTestCase { Expression exp = new ToLongExpression(); assertVerify(DataType.INT, exp, DataType.LONG); assertVerify(DataType.STRING, exp, DataType.LONG); - assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java index 99d4fc2731f..1317df130ff 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java @@ -31,8 +31,8 @@ public class ToPositionTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new ToPositionExpression(); assertVerify(DataType.STRING, exp, PositionDataType.INSTANCE); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java index 43dfbf76b7f..6c3bc8fc160 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java @@ -30,7 +30,7 @@ public class ToStringTestCase { Expression exp = new ToStringExpression(); assertVerify(DataType.INT, exp, DataType.STRING); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java index 3b284fd9ae9..c0c938bb3b6 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java @@ -60,7 +60,7 @@ public class ToWsetTestCase { DataType.getWeightedSet(DataType.INT, createIfNonExistent, removeIfZero)); ExpressionAssert.assertVerify(DataType.STRING, exp, DataType.getWeightedSet(DataType.STRING, createIfNonExistent, removeIfZero)); - assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); } private static void assertConvert(boolean createIfNonExistent, boolean removeIfZero) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java index 309c57533cc..a92205cc30f 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java @@ -48,8 +48,8 @@ public class TokenizeTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig()); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java index 5664349a96e..95c272e1fca 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java @@ -28,8 +28,8 @@ public class TrimTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new TrimExpression(); assertVerify(DataType.STRING, exp, DataType.STRING); - assertVerifyThrows(null, exp, "Expected string input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveTestCase.java index ee6a5793d76..b0899d209cb 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveTestCase.java @@ -30,8 +30,8 @@ public class ZCurveTestCase { public void requireThatExpressionCanBeVerified() { Expression exp = new ZCurveExpression(); assertVerify(PositionDataType.INSTANCE, exp, DataType.LONG); - assertVerifyThrows(null, exp, "Expected position input, got null."); - assertVerifyThrows(DataType.INT, exp, "Expected position input, got int."); + assertVerifyThrows(null, exp, "Expected position input, but no input is specified"); + assertVerifyThrows(DataType.INT, exp, "Expected position input, got int"); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java index 551f770a01e..45438a8f273 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java @@ -51,7 +51,7 @@ public class ExpressionTestCase { assertExpression(ScriptExpression.class, "{ 1; 2 }"); assertExpression(SelectInputExpression.class, "select_input { field1: 2; }"); assertExpression(SetLanguageExpression.class, "set_language"); - assertExpression(SetValueExpression.class, "1"); + assertExpression(ConstantExpression.class, "1"); assertExpression(SetVarExpression.class, "set_var myvar1"); assertExpression(SplitExpression.class, "split '1'"); assertExpression(StatementExpression.class, "1 | 2"); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/FieldNameTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/FieldNameTestCase.java index 41b8731995e..d021a633a36 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/FieldNameTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/FieldNameTestCase.java @@ -23,7 +23,7 @@ public class FieldNameTestCase { public void requireThatCatDotIsNotConfusedWithFieldName() throws ParseException { assertEquals(new CatExpression(new InputExpression("foo"), new InputExpression("bar")), Expression.fromString("input foo . input bar")); - assertEquals(new CatExpression(new InputExpression("foo"), new SetValueExpression(new StringFieldValue("bar"))), + assertEquals(new CatExpression(new InputExpression("foo"), new ConstantExpression(new StringFieldValue("bar"))), Expression.fromString("input foo . 'bar'")); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ScriptTestCase.java index ad2b9ebd6f9..978c827556b 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ScriptTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.indexinglanguage.parser; +import com.yahoo.vespa.indexinglanguage.expressions.ConstantExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import org.junit.Test; @@ -17,7 +17,7 @@ public class ScriptTestCase { @Test public void requireThatRootProductionIsFlexible() throws ParseException { - assertRoot(SetValueExpression.class, "1"); + assertRoot(ConstantExpression.class, "1"); assertRoot(StatementExpression.class, "1 | echo"); assertRoot(StatementExpression.class, "{ 1 | echo }"); assertRoot(StatementExpression.class, "{ 1 | echo; }"); diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneHandler.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneHandler.java new file mode 100644 index 00000000000..09cf2abdbd3 --- /dev/null +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneHandler.java @@ -0,0 +1,55 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.filter.security.cloud; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudTokenDataPlaneFilterConfig; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudTokenDataPlaneFilterConfig.Clients.Tokens; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.Executor; + +import static java.util.stream.Collectors.flatMapping; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.toCollection; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; + +/** + * @author jonmv + */ +public class CloudTokenDataPlaneHandler extends ThreadedHttpRequestHandler { + + private final Map<String, Set<String>> tokens; + + @Inject + public CloudTokenDataPlaneHandler(CloudTokenDataPlaneFilterConfig config, Executor executor) { + super(executor); + tokens = new TreeMap<>(config.clients().stream() + .flatMap(client -> client.tokens().stream()) + .collect(groupingBy(Tokens::id, + flatMapping(token -> token.fingerprints().stream(), + toCollection(TreeSet::new))))); + } + + @Override + public HttpResponse handle(HttpRequest request) { + return new SlimeJsonResponse() {{ + Cursor tokensArray = slime.setObject().setArray("tokens"); + tokens.forEach((id, fingerprints) -> { + Cursor tokenObject = tokensArray.addObject(); + tokenObject.setString("id", id); + fingerprints.forEach(tokenObject.setArray("fingerprints")::addString); + }); + }}; + } + +} diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneHandlerTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneHandlerTest.java new file mode 100644 index 00000000000..c066dae6dca --- /dev/null +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneHandlerTest.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.filter.security.cloud; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudTokenDataPlaneFilterConfig.Builder; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudTokenDataPlaneFilterConfig.Clients; +import com.yahoo.jdisc.http.filter.security.cloud.config.CloudTokenDataPlaneFilterConfig.Clients.Tokens; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; + +import static com.yahoo.container.jdisc.HttpRequest.createTestRequest; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author jonmv + */ +public class CloudTokenDataPlaneHandlerTest { + + @Test + void testFingerprints() throws IOException { + CloudTokenDataPlaneHandler handler = new CloudTokenDataPlaneHandler( + new Builder().tokenContext("context") + .clients(new Clients.Builder().id("client1") + .permissions("read") + .tokens(new Tokens.Builder().id("id1") + .fingerprints(List.of("pinky", "ring", "middle", "index", "thumb")) + .checkAccessHashes(List.of("a", "b", "c", "d", "e")) + .expirations(List.of("<none>", "<none>", "<none>", "<none>", "<none>"))) + .tokens(new Tokens.Builder().id("id2") + .fingerprints("toasty") + .checkAccessHashes("hash") + .expirations("<none>"))) + .clients(new Clients.Builder().id("client2") + .permissions("write") + .tokens(new Tokens.Builder().id("id2") + .fingerprints("toasty") + .checkAccessHashes("hash") + .expirations("<none>"))) + .build(), + Runnable::run + ); + + HttpResponse response = handler.handle(createTestRequest("", GET)); + assertEquals(200, + response.getStatus()); + assertEquals(""" + {"tokens":[{"id":"id1","fingerprints":["index","middle","pinky","ring","thumb"]},{"id":"id2","fingerprints":["toasty"]}]}""", + new ByteArrayOutputStream() {{ response.render(this); }}.toString(UTF_8)); + } + +} diff --git a/lucene-linguistics/src/main/java/com/yahoo/language/lucene/AnalyzerFactory.java b/lucene-linguistics/src/main/java/com/yahoo/language/lucene/AnalyzerFactory.java index 67a430a28dc..92ea77cdc13 100644 --- a/lucene-linguistics/src/main/java/com/yahoo/language/lucene/AnalyzerFactory.java +++ b/lucene-linguistics/src/main/java/com/yahoo/language/lucene/AnalyzerFactory.java @@ -11,9 +11,9 @@ import org.apache.lucene.analysis.custom.CustomAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import java.io.IOException; -import java.nio.file.Path; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Logger; @@ -60,13 +60,15 @@ class AnalyzerFactory { } private Analyzer createAnalyzer(AnalyzerKey analyzerKey) { - if (null != config.analysis(analyzerKey.languageCode())) { + LuceneAnalysisConfig.Analysis analysis = analysisConfig(analyzerKey); + if (null != analysis) { log.config("Creating analyzer for " + analyzerKey + " from config"); - return createAnalyzer(analyzerKey, config.analysis(analyzerKey.languageCode())); + return createAnalyzer(analyzerKey, analysis); } - if (null != analyzerComponents.getComponent(analyzerKey.languageCode())) { + Analyzer analyzerFromComponents = fromComponents(analyzerKey); + if (null != analyzerFromComponents) { log.config("Using analyzer for " + analyzerKey + " from components"); - return analyzerComponents.getComponent(analyzerKey.languageCode()); + return analyzerFromComponents; } if (null != defaultAnalyzers.get(analyzerKey.language())) { log.config("Using Analyzer for " + analyzerKey + " from a list of default language analyzers"); @@ -77,6 +79,24 @@ class AnalyzerFactory { return defaultAnalyzer; } + /** + * First, checks if more specific (language + stemMode) analysis is configured. + * Second, checks if analysis is configured only for a languageCode. + */ + private LuceneAnalysisConfig.Analysis analysisConfig(AnalyzerKey analyzerKey) { + LuceneAnalysisConfig.Analysis analysis = config.analysis(analyzerKey.languageCodeAndStemMode()); + return (null != analysis) ? analysis : config.analysis(analyzerKey.languageCode()); + } + + /** + * First, checks if a component is configured for a languageCode + StemMode. + * Second, checks if Analyzer is configured only for a languageCode. + */ + private Analyzer fromComponents(AnalyzerKey analyzerKey) { + Analyzer analyzer = analyzerComponents.getComponent(analyzerKey.languageCodeAndStemMode()); + return (null != analyzer) ? analyzer : analyzerComponents.getComponent(analyzerKey.languageCode()); + } + private Analyzer createAnalyzer(AnalyzerKey analyzerKey, LuceneAnalysisConfig.Analysis analysis) { try { CustomAnalyzer.Builder builder = config.configDir() @@ -143,9 +163,14 @@ class AnalyzerFactory { private record AnalyzerKey(Language language, StemMode stemMode, boolean removeAccents) { - // TODO: Identity here is determined by language only. - // Would it make sense to combine language + stemMode + removeAccents to make - // a composite key so we can have more variations possible? + /** + * Combines the languageCode and the stemMode. + * It allows to specify up to 6 (5 StemModes and only language code) analyzers per language. + * The `/` is used so that it doesn't conflict with ComponentRegistry keys. + */ + public String languageCodeAndStemMode() { + return language.languageCode() + "/" + stemMode.toString(); + } public String languageCode() { return language.languageCode(); @@ -155,12 +180,12 @@ class AnalyzerFactory { public boolean equals(Object o) { if (o == this) return true; if ( ! (o instanceof AnalyzerKey other)) return false; - return other.language == this.language; + return other.language == this.language && other.stemMode == this.stemMode; } @Override public int hashCode() { - return language.hashCode(); + return Objects.hash(language, stemMode); } } diff --git a/lucene-linguistics/src/test/java/com/yahoo/language/lucene/LuceneTokenizerTest.java b/lucene-linguistics/src/test/java/com/yahoo/language/lucene/LuceneTokenizerTest.java index 92c369bc60c..fc29fcc0071 100644 --- a/lucene-linguistics/src/test/java/com/yahoo/language/lucene/LuceneTokenizerTest.java +++ b/lucene-linguistics/src/test/java/com/yahoo/language/lucene/LuceneTokenizerTest.java @@ -197,4 +197,33 @@ public class LuceneTokenizerTest { .tokenize("Dogs and Cats", Language.ENGLISH, StemMode.ALL, false); assertEquals(List.of("and", "Cat"), tokenStrings(tokens)); } + + @Test + public void compositeConfigKey() { + String reversingAnalyzerKey = Language.ENGLISH.languageCode() + + "/" + + StemMode.ALL; + LuceneAnalysisConfig enConfig = new LuceneAnalysisConfig.Builder() + .analysis( + Map.of(reversingAnalyzerKey, + new LuceneAnalysisConfig.Analysis.Builder().tokenFilters(List.of( + new LuceneAnalysisConfig + .Analysis + .TokenFilters + .Builder() + .name("reverseString")))) + ).build(); + LuceneLinguistics linguistics = new LuceneLinguistics(enConfig, new ComponentRegistry<>()); + // Matching StemMode + Iterable<Token> tokens = linguistics + .getTokenizer() + .tokenize("Dogs and Cats", Language.ENGLISH, StemMode.ALL, false); + assertEquals(List.of("sgoD", "dna", "staC"), tokenStrings(tokens)); + // StemMode is different + Iterable<Token> stemModeTokens = linguistics + .getTokenizer() + .tokenize("Dogs and Cats", Language.ENGLISH, StemMode.BEST, false); + assertEquals(List.of("dog", "cat"), tokenStrings(stemModeTokens)); + + } } diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index 970bb6732a1..06f2f34964b 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -1,41 +1,31 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#[non-test] -# Contains dependencies that are not used exclusively in 'test' scope -aopalliance:aopalliance:1.0 -com.fasterxml.jackson.core:jackson-annotations:2.15.2 -com.fasterxml.jackson.core:jackson-core:2.15.2 -com.fasterxml.jackson.core:jackson-databind:2.15.2 -com.github.luben:zstd-jni:1.5.5-5 -com.google.errorprone:error_prone_annotations:2.21.1 +aopalliance:aopalliance:${aopalliance.vespa.version} +com.fasterxml.jackson.core:jackson-annotations:${jackson2.vespa.version} +com.fasterxml.jackson.core:jackson-core:${jackson2.vespa.version} +com.fasterxml.jackson.core:jackson-databind:${jackson-databind.vespa.version} +com.github.luben:zstd-jni:${luben.zstd.vespa.version} +com.google.errorprone:error_prone_annotations:${error-prone-annotations.vespa.version} com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:32.1.2-jre -com.google.inject:guice:6.0.0 +com.google.guava:guava:${guava.vespa.version} +com.google.inject:guice:${guice.vespa.version} com.google.j2objc:j2objc-annotations:2.8 -commons-codec:commons-codec:1.16.0 -commons-io:commons-io:2.13.0 -jakarta.inject:jakarta.inject-api:2.0.1 -javax.annotation:javax.annotation-api:1.2 -javax.inject:javax.inject:1 +commons-codec:commons-codec:${commons-codec.vespa.version} +commons-io:commons-io:${commons-io.vespa.version} +jakarta.inject:jakarta.inject-api:${jakarta.inject.vespa.version} +javax.annotation:javax.annotation-api:${commons-logging.vespa.version} +javax.inject:javax.inject:${javax.inject.vespa.version} +junit:junit:${junit4.vespa.version} +net.bytebuddy:byte-buddy-agent:${byte-buddy.vespa.version} +net.bytebuddy:byte-buddy:${byte-buddy.vespa.version} org.apache-extras.beanshell:bsh:2.0b6 org.apache.commons:commons-collections4:4.4 -org.apache.commons:commons-compress:1.24.0 -org.apache.commons:commons-lang3:3.13.0 -org.apache.maven:maven-archiver:3.6.1 -org.apache.maven:maven-artifact:3.9.4 -org.apache.maven:maven-builder-support:3.9.4 -org.apache.maven:maven-core:3.9.4 -org.apache.maven:maven-model:3.9.4 -org.apache.maven:maven-model-builder:3.9.4 -org.apache.maven:maven-plugin-api:3.9.4 -org.apache.maven:maven-repository-metadata:3.9.4 -org.apache.maven:maven-resolver-provider:3.9.4 -org.apache.maven:maven-settings:3.9.4 -org.apache.maven:maven-settings-builder:3.9.4 -org.apache.maven.enforcer:enforcer-api:3.4.1 -org.apache.maven.enforcer:enforcer-rules:3.4.1 -org.apache.maven.plugin-tools:maven-plugin-annotations:3.9.0 -org.apache.maven.plugins:maven-shade-plugin:3.5.0 +org.apache.commons:commons-compress:${commons-compress.vespa.version} +org.apache.commons:commons-lang3:${commons-lang3.vespa.version} +org.apache.maven.enforcer:enforcer-api:${maven-enforcer-plugin.vespa.version} +org.apache.maven.enforcer:enforcer-rules:${maven-enforcer-plugin.vespa.version} +org.apache.maven.plugin-tools:maven-plugin-annotations:${maven-plugin-tools.vespa.version} +org.apache.maven.plugins:maven-shade-plugin:${maven-shade-plugin.vespa.version} org.apache.maven.resolver:maven-resolver-api:1.9.14 org.apache.maven.resolver:maven-resolver-impl:1.9.14 org.apache.maven.resolver:maven-resolver-named-locks:1.9.14 @@ -43,42 +33,47 @@ org.apache.maven.resolver:maven-resolver-spi:1.9.14 org.apache.maven.resolver:maven-resolver-util:1.9.14 org.apache.maven.shared:maven-dependency-tree:3.2.1 org.apache.maven.shared:maven-shared-utils:3.3.4 +org.apache.maven:maven-archiver:${maven-archiver.vespa.version} +org.apache.maven:maven-artifact:${maven-core.vespa.version} +org.apache.maven:maven-builder-support:${maven-core.vespa.version} +org.apache.maven:maven-core:${maven-core.vespa.version} +org.apache.maven:maven-model-builder:${maven-core.vespa.version} +org.apache.maven:maven-model:${maven-core.vespa.version} +org.apache.maven:maven-plugin-api:${maven-plugin-api.vespa.version} +org.apache.maven:maven-repository-metadata:${maven-core.vespa.version} +org.apache.maven:maven-resolver-provider:${maven-core.vespa.version} +org.apache.maven:maven-settings-builder:${maven-core.vespa.version} +org.apache.maven:maven-settings:${maven-core.vespa.version} +org.apiguardian:apiguardian-api:${apiguardian.vespa.version} org.codehaus.plexus:plexus-archiver:4.8.0 org.codehaus.plexus:plexus-cipher:2.0 org.codehaus.plexus:plexus-classworlds:2.7.0 org.codehaus.plexus:plexus-component-annotations:2.1.0 org.codehaus.plexus:plexus-interpolation:1.26 -org.codehaus.plexus:plexus-io:3.4.1 +org.codehaus.plexus:plexus-io:${maven-enforcer-plugin.vespa.version} org.codehaus.plexus:plexus-sec-dispatcher:2.0 -org.codehaus.plexus:plexus-utils:3.5.1 +org.codehaus.plexus:plexus-utils:${maven-shade-plugin.vespa.version} org.eclipse.aether:aether-api:1.0.0.v20140518 org.eclipse.aether:aether-util:1.0.0.v20140518 org.eclipse.sisu:org.eclipse.sisu.inject:0.3.5 org.eclipse.sisu:org.eclipse.sisu.plexus:0.3.5 +org.hamcrest:hamcrest-core:${hamcrest.vespa.version} +org.hamcrest:hamcrest:${hamcrest.vespa.version} org.iq80.snappy:snappy:0.4 org.jdom:jdom2:2.0.6.1 -org.ow2.asm:asm:9.5 -org.ow2.asm:asm-commons:9.5 -org.ow2.asm:asm-tree:9.5 -org.slf4j:slf4j-api:1.7.36 -org.tukaani:xz:1.9 -org.twdata.maven:mojo-executor:2.4.0 -org.vafer:jdependency:2.8.0 - -#[test-only] -# Contains dependencies that are used exclusively in 'test' scope -junit:junit:4.13.2 -net.bytebuddy:byte-buddy:1.14.8 -net.bytebuddy:byte-buddy-agent:1.14.8 -org.apiguardian:apiguardian-api:1.1.2 -org.hamcrest:hamcrest:2.2 -org.hamcrest:hamcrest-core:2.2 -org.junit.jupiter:junit-jupiter:5.10.0 -org.junit.jupiter:junit-jupiter-api:5.10.0 -org.junit.jupiter:junit-jupiter-engine:5.10.0 -org.junit.jupiter:junit-jupiter-params:5.10.0 -org.junit.platform:junit-platform-commons:1.10.0 -org.junit.platform:junit-platform-engine:1.10.0 -org.mockito:mockito-core:5.5.0 +org.junit.jupiter:junit-jupiter-api:${junit.vespa.version} +org.junit.jupiter:junit-jupiter-engine:${junit.vespa.version} +org.junit.jupiter:junit-jupiter-params:${junit.vespa.version} +org.junit.jupiter:junit-jupiter:${junit.vespa.version} +org.junit.platform:junit-platform-commons:${junit.platform.vespa.version} +org.junit.platform:junit-platform-engine:${junit.platform.vespa.version} +org.mockito:mockito-core:${mockito.vespa.version} org.objenesis:objenesis:3.3 -org.opentest4j:opentest4j:1.3.0 +org.opentest4j:opentest4j:${opentest4j.vespa.version} +org.ow2.asm:asm-commons:${asm.vespa.version} +org.ow2.asm:asm-tree:${asm.vespa.version} +org.ow2.asm:asm:${asm.vespa.version} +org.slf4j:slf4j-api:${slf4j.vespa.version} +org.tukaani:xz:1.9 +org.twdata.maven:mojo-executor:${mojo-executor.vespa.version} +org.vafer:jdependency:2.9.0 diff --git a/maven-plugins/pom.xml b/maven-plugins/pom.xml index f12731ba9da..bc8ed090c3f 100644 --- a/maven-plugins/pom.xml +++ b/maven-plugins/pom.xml @@ -47,7 +47,7 @@ </goals> <configuration> <rules> - <enforceDependencies implementation="com.yahoo.vespa.maven.plugin.enforcer.EnforceDependenciesAllProjects"> + <enforceDependencies implementation="com.yahoo.vespa.maven.plugin.enforcer.AllowedDependencies"> <rootProjectId>com.yahoo.vespa:maven-plugins</rootProjectId> <specFile>allowed-maven-dependencies.txt</specFile> <ignored> diff --git a/metrics/src/main/java/ai/vespa/metrics/docs/DocumentationGenerator.java b/metrics/src/main/java/ai/vespa/metrics/docs/DocumentationGenerator.java new file mode 100644 index 00000000000..9d63d4af159 --- /dev/null +++ b/metrics/src/main/java/ai/vespa/metrics/docs/DocumentationGenerator.java @@ -0,0 +1,65 @@ +package ai.vespa.metrics.docs; + +import ai.vespa.metrics.ClusterControllerMetrics; +import ai.vespa.metrics.ConfigServerMetrics; +import ai.vespa.metrics.ContainerMetrics; +import ai.vespa.metrics.DistributorMetrics; +import ai.vespa.metrics.LogdMetrics; +import ai.vespa.metrics.NodeAdminMetrics; +import ai.vespa.metrics.SearchNodeMetrics; +import ai.vespa.metrics.SentinelMetrics; +import ai.vespa.metrics.SlobrokMetrics; +import ai.vespa.metrics.StorageMetrics; +import ai.vespa.metrics.Unit; +import ai.vespa.metrics.VespaMetrics; +import ai.vespa.metrics.set.DefaultMetrics; +import ai.vespa.metrics.set.MetricSet; +import ai.vespa.metrics.set.VespaMetricSet; + +import java.util.Map; +import static ai.vespa.metrics.docs.MetricDocumentation.writeMetricDocumentation; +import static ai.vespa.metrics.docs.MetricSetDocumentation.writeMetricSetDocumentation; + +/** + * @author olaa + * + * Helper class to generate metric reference documentation for docs.vespa.ai + */ +public class DocumentationGenerator { + + public static void main(String[] args) { + + if (args.length != 1) { + throw new IllegalArgumentException("Expected exactly one argument: directory to write to"); + } + var path = args[0]; + + var metrics = getMetrics(); + metrics.forEach((metricType, metricArray) -> writeMetricDocumentation(path, metricArray, metricType)); + + var metricSets = getMetricSets(); + metricSets.forEach((name, metricSet) -> writeMetricSetDocumentation(path, name, metricSet, metrics)); + + UnitDocumentation.writeUnitDocumentation(path, Unit.values()); + } + + private static Map<String, VespaMetrics[]> getMetrics() { + return Map.of( + "Container", ContainerMetrics.values(), + "SearchNode", SearchNodeMetrics.values(), + "Storage", StorageMetrics.values(), + "Distributor", DistributorMetrics.values(), + "ConfigServer", ConfigServerMetrics.values(), + "Logd", LogdMetrics.values(), + "NodeAdmin", NodeAdminMetrics.values(), + "Slobrok", SlobrokMetrics.values(), + "Sentinel", SentinelMetrics.values(), + "ClusterController", ClusterControllerMetrics.values() + ); + } + + private static Map<String, MetricSet> getMetricSets() { + return Map.of("Vespa", VespaMetricSet.vespaMetricSet, + "Default", DefaultMetrics.defaultMetricSet); + } +} diff --git a/metrics/src/main/java/ai/vespa/metrics/docs/MetricDocumentation.java b/metrics/src/main/java/ai/vespa/metrics/docs/MetricDocumentation.java new file mode 100644 index 00000000000..fc46c785f0e --- /dev/null +++ b/metrics/src/main/java/ai/vespa/metrics/docs/MetricDocumentation.java @@ -0,0 +1,57 @@ +package ai.vespa.metrics.docs; + +import ai.vespa.metrics.VespaMetrics; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @author olaa + */ +public class MetricDocumentation { + + protected static void writeMetricDocumentation(String path, VespaMetrics[] metrics, String metricType) { + var referenceBuilder = new StringBuilder(); + referenceBuilder.append(String.format(""" + --- + # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + title: "%s Metrics" + --- + + <table class="table"> + <thead> + <tr><th>Name</th><th>Description</th><th>Unit</th></tr> + </thead> + <tbody> + %s </tbody> + </table> + """, metricType, htmlRows(metrics))); + + try (FileWriter fileWriter = new FileWriter(path + "/" + metricType.toLowerCase() + "-metrics-reference.html")) { + fileWriter.write(referenceBuilder.toString()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static String htmlRows(VespaMetrics[] metrics) { + return Stream.of(metrics) + .map(metric -> + String.format( + """ + <tr> + <td><p id="%s">%s</p></td> + <td>%s</td> + <td>%s</td> + </tr> + """, + metric.baseName().replaceAll("\\.", "_"), + metric.baseName(), + metric.description(), + metric.unit().toString().toLowerCase()) + ).collect(Collectors.joining()); + } +} diff --git a/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java b/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java new file mode 100644 index 00000000000..bafefbfedd6 --- /dev/null +++ b/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java @@ -0,0 +1,105 @@ +package ai.vespa.metrics.docs; + +import ai.vespa.metrics.Suffix; +import ai.vespa.metrics.VespaMetrics; +import ai.vespa.metrics.set.MetricSet; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author olaa + */ +public class MetricSetDocumentation { + + protected static void writeMetricSetDocumentation(String path, String name, MetricSet metricSet, Map<String, VespaMetrics[]> metricsByType) { + + var groupedBySuffix = metricSet.getMetrics() + .keySet() + .stream() + .map(MetricSetDocumentation::withSuffix) + .collect(Collectors.groupingBy(Map.Entry::getKey, Collectors.mapping(Map.Entry::getValue, Collectors.toCollection(LinkedHashSet::new)))); + + var metricTypeByName = metricsByType.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, + entry -> Arrays.stream(entry.getValue()) + .filter(val -> groupedBySuffix.containsKey(val.baseName())) + .collect(Collectors.toMap( + val -> val, + val -> groupedBySuffix.get(val.baseName()), + (a, b) -> a, + LinkedHashMap::new + )), + (a, b) -> a, + LinkedHashMap::new)); + + var referenceBuilder = new StringBuilder(); + referenceBuilder.append(String.format(""" + --- + # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + title: "%s Metric Set" + ---""", name)); + metricsByType.keySet() + .stream() + .sorted() + .filter(m -> !metricTypeByName.get(m).isEmpty()) + .forEach(type -> + referenceBuilder.append(String.format(""" + + <h2 id="%s-metrics">%s Metrics</h2> + <table class="table"> + <thead> + <tr><th>Name</th><th>Description</th><th>Unit</th><th>Suffixes</th></tr> + </thead> + <tbody> + %s </tbody> + </table> + """, type.toLowerCase(), type, htmlRows(metricTypeByName.get(type)))) + ); + try (FileWriter fileWriter = new FileWriter(path + "/" + metricSet.getId().toLowerCase() + "-set-metrics-reference.html")) { + fileWriter.write(referenceBuilder.toString()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static String htmlRows(Map<VespaMetrics, LinkedHashSet<String>> metrics) { + return metrics.entrySet() + .stream() + .map(entry -> + String.format( + """ + <tr> + <td><p id="%s">%s</p></td> + <td>%s</td> + <td>%s</td> + <td>%s</td> + </tr> + """, + entry.getKey().baseName().replaceAll("\\.", "_"), + entry.getKey().baseName(), + entry.getKey().description(), + entry.getKey().unit().toString().toLowerCase(), + String.join(", ", entry.getValue())) + + ).collect(Collectors.joining()); + } + + private static Map.Entry<String, String> withSuffix(String metricName) { + try { + var suffixIndex = metricName.lastIndexOf("."); + var suffix = Suffix.valueOf(metricName.substring(suffixIndex + 1)); + return Map.entry(metricName.substring(0, suffixIndex), suffix.toString()); + } catch (Exception e) { + return Map.entry(metricName, "N/A"); + } + } + +} diff --git a/metrics/src/main/java/ai/vespa/metrics/docs/UnitDocumentation.java b/metrics/src/main/java/ai/vespa/metrics/docs/UnitDocumentation.java new file mode 100644 index 00000000000..03f99a076b1 --- /dev/null +++ b/metrics/src/main/java/ai/vespa/metrics/docs/UnitDocumentation.java @@ -0,0 +1,56 @@ +package ai.vespa.metrics.docs; + + +import ai.vespa.metrics.Unit; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @author olaa + */ +public class UnitDocumentation { + + protected static void writeUnitDocumentation(String path, Unit[] units) { + var referenceBuilder = new StringBuilder(); + referenceBuilder.append(String.format(""" + --- + # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + title: "Metric Units Reference" + --- + + + <table class="table"> + <thead> + <tr><th>Unit</th><th>Description</th></tr> + </thead> + <tbody> + %s </tbody> + </table> + """, htmlRows(units))); + + try (FileWriter fileWriter = new FileWriter(path + "/unit-metrics-reference.html")) { + fileWriter.write(referenceBuilder.toString()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static String htmlRows(Unit[] units) { + return Stream.of(units) + .map(unit -> + String.format( + """ + <tr> + <td>%s</td> + <td>%s</td> + </tr> + """, + unit.fullName(), + unit.getDescription()) + ).collect(Collectors.joining()); + } +} diff --git a/model-integration/src/main/java/ai/vespa/embedding/ColBertEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/ColBertEmbedder.java index aafb9877c27..4bb7bcc9225 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/ColBertEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/ColBertEmbedder.java @@ -27,7 +27,7 @@ import java.util.Arrays; import static com.yahoo.language.huggingface.ModelInfo.TruncationStrategy.LONGEST_FIRST; /** - * A ColBERT embedder implementation that maps text to multiple vectors, one vector per subword id. + * A ColBERT embedder implementation that maps text to multiple vectors, one vector per token subword id. * This embedder uses a HuggingFace tokenizer to produce a token sequence that is then input to a transformer model. * * See col-bert-embedder.def for configurable parameters. @@ -60,10 +60,8 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { attentionMaskName = config.transformerAttentionMask(); outputName = config.transformerOutput(); maxTransformerTokens = config.transformerMaxTokens(); - if(config.maxDocumentTokens() > maxTransformerTokens) - throw new IllegalArgumentException("maxDocumentTokens must be less than or equal to transformerMaxTokens"); - maxDocumentTokens = config.maxDocumentTokens(); - maxQueryTokens = config.maxQueryTokens(); + maxDocumentTokens = Math.min(config.maxDocumentTokens(), maxTransformerTokens); + maxQueryTokens = Math.min(config.maxQueryTokens(), maxTransformerTokens); startSequenceToken = config.transformerStartSequenceToken(); endSequenceToken = config.transformerEndSequenceToken(); maskSequenceToken = config.transformerMaskToken(); @@ -75,7 +73,8 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { .setPadding(false); var info = HuggingFaceTokenizer.getModelInfo(tokenizerPath); if (info.maxLength() == -1 || info.truncation() != LONGEST_FIRST) { - // Force truncation to max token vector length accepted by model if tokenizer.json contains no valid truncation configuration + // Force truncation + // to max length accepted by model if tokenizer.json contains no valid truncation configuration int maxLength = info.maxLength() > 0 && info.maxLength() <= config.transformerMaxTokens() ? info.maxLength() : config.transformerMaxTokens(); @@ -115,8 +114,8 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { @Override public Tensor embed(String text, Context context, TensorType tensorType) { if(!verifyTensorType(tensorType)) { - throw new IllegalArgumentException("Invalid ColBERT embedder tensor destination." + - "Wanted a mixed 2-d mapped-indexed tensor, got " + tensorType.toString()); + throw new IllegalArgumentException("Invalid ColBERT embedder tensor destination. " + + "Wanted a mixed 2-d mapped-indexed tensor, got " + tensorType); } if (context.getDestination().startsWith("query")) { return embedQuery(text, context, tensorType); @@ -152,6 +151,7 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { inputIds.add(Q_TOKEN_ID); inputIds.addAll(ids); inputIds.add(endSequenceToken); + int length = inputIds.size(); int padding = maxQueryTokens - length; @@ -177,12 +177,9 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { throw new IllegalArgumentException("Token dimensionality does not" + " match indexed dimensionality of " + dims); } - Tensor.Builder builder = Tensor.Builder.of(tensorType); - for (int token = 0; token < result.shape()[0]; token++) - for (int d = 0; d < result.shape()[1]; d++) - builder.cell(TensorAddress.of(token, d), result.get(TensorAddress.of(token, d))); + Tensor resultTensor = toFloatTensor(result, tensorType, inputIds.size()); runtime.sampleEmbeddingLatency((System.nanoTime() - start) / 1_000_000d, context); - return builder.build(); + return resultTensor; } protected Tensor embedDocument(String text, Context context, TensorType tensorType) { @@ -193,7 +190,6 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { List<Long> ids = encoding.ids().stream().filter(token -> !PUNCTUATION_TOKEN_IDS.contains(token)).toList(); - ; if (ids.size() > maxDocumentTokens - 3) ids = ids.subList(0, maxDocumentTokens - 3); @@ -216,29 +212,29 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { Tensor tokenEmbeddings = outputs.get(outputName); IndexedTensor result = (IndexedTensor) tokenEmbeddings.reduce(Reduce.Aggregator.min, "d0"); Tensor contextualEmbeddings; + int retainedTokens = inputIds.size() -1; //Do not retain last PAD if(tensorType.valueType() == TensorType.Value.INT8) { - contextualEmbeddings = toBitTensor(result, tensorType); + contextualEmbeddings = toBitTensor(result, tensorType, retainedTokens); } else { - contextualEmbeddings = toFloatTensor(result, tensorType); + contextualEmbeddings = toFloatTensor(result, tensorType, retainedTokens); } - runtime.sampleEmbeddingLatency((System.nanoTime() - start) / 1_000_000d, context); return contextualEmbeddings; } - public static Tensor toFloatTensor(IndexedTensor result, TensorType type) { + public static Tensor toFloatTensor(IndexedTensor result, TensorType type, int nTokens) { int size = type.indexedSubtype().dimensions().size(); if (size != 1) throw new IllegalArgumentException("Indexed tensor must have one dimension"); - int dims = type.indexedSubtype().dimensions().get(0).size().get().intValue(); - int resultDim = (int)result.shape()[1]; - if(resultDim != dims) { - throw new IllegalArgumentException("Not possible to map token vector embedding with " + resultDim - + " + dimensions into tensor with " + dims); + int wantedDimensionality = type.indexedSubtype().dimensions().get(0).size().get().intValue(); + int resultDimensionality = (int)result.shape()[1]; + if(resultDimensionality != wantedDimensionality) { + throw new IllegalArgumentException("Not possible to map token vector embedding with " + resultDimensionality + + " + dimensions into tensor with " + wantedDimensionality); } Tensor.Builder builder = Tensor.Builder.of(type); - for (int token = 0; token < result.shape()[0]; token++) { - for (int d = 0; d < result.shape()[1]; d++) { + for (int token = 0; token < nTokens; token++) { + for (int d = 0; d < resultDimensionality; d++) { var value = result.get(TensorAddress.of(token, d)); builder.cell(TensorAddress.of(token,d),value); } @@ -246,21 +242,21 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { return builder.build(); } - public static Tensor toBitTensor(IndexedTensor result, TensorType type) { + public static Tensor toBitTensor(IndexedTensor result, TensorType type, int nTokens) { if (type.valueType() != TensorType.Value.INT8) throw new IllegalArgumentException("Only a int8 tensor type can be" + " the destination of bit packing"); int size = type.indexedSubtype().dimensions().size(); if (size != 1) throw new IllegalArgumentException("Indexed tensor must have one dimension"); - int dims = type.indexedSubtype().dimensions().get(0).size().get().intValue(); - int resultDim = (int)result.shape()[1]; - if(resultDim/8 != dims) { - throw new IllegalArgumentException("Not possible to pack " + resultDim - + " + dimensions into " + dims); + int wantedDimensionality = type.indexedSubtype().dimensions().get(0).size().get().intValue(); + int resultDimensionality = (int)result.shape()[1]; + if(resultDimensionality/8 != wantedDimensionality) { + throw new IllegalArgumentException("Not possible to pack " + resultDimensionality + + " + dimensions into " + wantedDimensionality + " dimensions"); } Tensor.Builder builder = Tensor.Builder.of(type); - for (int token = 0; token < result.shape()[0]; token++) { + for (int token = 0; token < nTokens; token++) { BitSet bitSet = new BitSet(8); int key = 0; for (int d = 0; d < result.shape()[1]; d++) { diff --git a/model-integration/src/test/java/ai/vespa/embedding/ColBertEmbedderTest.java b/model-integration/src/test/java/ai/vespa/embedding/ColBertEmbedderTest.java index 8516f6e6689..4e398f7245d 100644 --- a/model-integration/src/test/java/ai/vespa/embedding/ColBertEmbedderTest.java +++ b/model-integration/src/test/java/ai/vespa/embedding/ColBertEmbedderTest.java @@ -31,7 +31,7 @@ public class ColBertEmbedderTest { "[1, 1, 1, 1, 1, 1, 1, 1]" + "]", TensorType.fromSpec("tensor<int8>(dt{},x[1])"), - "tensor<int8>(dt{},x[1]):{0:1.0, 1:5.0, 2:3.0, 3:127.0, 4:-128.0, 5:-1.0}" + "tensor<int8>(dt{},x[1]):{0:1.0, 1:5.0, 2:3.0, 3:127.0, 4:-128.0, 5:-1.0}", 6 ); assertPackedRight( "" + @@ -41,7 +41,7 @@ public class ColBertEmbedderTest { "[0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1]" + "]", TensorType.fromSpec("tensor<int8>(dt{},x[2])"), - "tensor<int8>(dt{},x[2]):{0:[1.0, -128.0], 1:[5.0, 1.0]}" + "tensor<int8>(dt{},x[2]):{0:[1.0, -128.0], 1:[5.0, 1.0]}",2 ); } @@ -75,18 +75,18 @@ public class ColBertEmbedderTest { } String text = sb.toString(); Tensor fullFloat = assertEmbed("tensor<float>(dt{},x[128])", text, indexingContext); - assertEquals(512*128,fullFloat.size()); + assertEquals(511*128,fullFloat.size()); Tensor query = assertEmbed("tensor<float>(dt{},x[128])", text, queryContext); assertEquals(32*128,query.size()); Tensor binaryRep = assertEmbed("tensor<int8>(dt{},x[16])", text, indexingContext); - assertEquals(512*16,binaryRep.size()); + assertEquals(511*16,binaryRep.size()); Tensor shortDoc = assertEmbed("tensor<int8>(dt{},x[16])", "annoyance", indexingContext); - // 4 tokens, 16 bytes each = 64 bytes - //because of CLS, special, sequence, SEP - assertEquals(4*16,shortDoc.size());; + // 3 tokens, 16 bytes each = 48 bytes + //CLS [unused1] sequence + assertEquals(3*16,shortDoc.size());; } static Tensor assertEmbed(String tensorSpec, String text, Embedder.Context context) { @@ -100,8 +100,8 @@ public class ColBertEmbedderTest { return result; } - static void assertPackedRight(String numbers, TensorType destination,String expected) { - Tensor packed = ColBertEmbedder.toBitTensor((IndexedTensor) Tensor.from(numbers), destination); + static void assertPackedRight(String numbers, TensorType destination,String expected, int size) { + Tensor packed = ColBertEmbedder.toBitTensor((IndexedTensor) Tensor.from(numbers), destination, size); assertEquals(expected,packed.toString()); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index 0300d7e92ff..d902fb7b3c4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; import java.net.URI; @@ -73,9 +74,7 @@ public class NodeSpec { private final List<TrustStoreItem> trustStore; - private final Optional<WireguardKey> wireguardPubkey; - - private final Optional<Instant> wireguardKeyTimestamp; + private final Optional<WireguardKeyWithTimestamp> wireguardKeyWithTimestamp; private final boolean wantToRebuild; @@ -112,8 +111,7 @@ public class NodeSpec { Optional<URI> archiveUri, Optional<ApplicationId> exclusiveTo, List<TrustStoreItem> trustStore, - Optional<WireguardKey> wireguardPubkey, - Optional<Instant> wireguardKeyTimestamp, + Optional<WireguardKeyWithTimestamp> wireguardPubkey, boolean wantToRebuild) { if (state == NodeState.active) { @@ -157,8 +155,7 @@ public class NodeSpec { this.archiveUri = Objects.requireNonNull(archiveUri); this.exclusiveTo = Objects.requireNonNull(exclusiveTo); this.trustStore = Objects.requireNonNull(trustStore); - this.wireguardPubkey = Objects.requireNonNull(wireguardPubkey); - this.wireguardKeyTimestamp = Objects.requireNonNull(wireguardKeyTimestamp); + this.wireguardKeyWithTimestamp = Objects.requireNonNull(wireguardPubkey); this.wantToRebuild = wantToRebuild; } @@ -313,9 +310,7 @@ public class NodeSpec { return trustStore; } - public Optional<WireguardKey> wireguardPubkey() { return wireguardPubkey; } - - public Optional<Instant> wireguardKeyTimestamp() { return wireguardKeyTimestamp; } + public Optional<WireguardKeyWithTimestamp> wireguardKeyWithTimestamp() { return wireguardKeyWithTimestamp; } public boolean wantToRebuild() { return wantToRebuild; @@ -358,8 +353,7 @@ public class NodeSpec { Objects.equals(archiveUri, that.archiveUri) && Objects.equals(exclusiveTo, that.exclusiveTo) && Objects.equals(trustStore, that.trustStore) && - Objects.equals(wireguardPubkey, that.wireguardPubkey) && - Objects.equals(wireguardKeyTimestamp, that.wireguardKeyTimestamp) && + Objects.equals(wireguardKeyWithTimestamp, that.wireguardKeyWithTimestamp) && Objects.equals(wantToRebuild, that.wantToRebuild); } @@ -398,8 +392,7 @@ public class NodeSpec { archiveUri, exclusiveTo, trustStore, - wireguardPubkey, - wireguardKeyTimestamp, + wireguardKeyWithTimestamp, wantToRebuild); } @@ -438,8 +431,7 @@ public class NodeSpec { + " archiveUri=" + archiveUri + " exclusiveTo=" + exclusiveTo + " trustStore=" + trustStore - + " wireguardPubkey=" + wireguardPubkey - + " wireguardKeyTimestamp=" + wireguardKeyTimestamp + + " wireguardPubkey=" + wireguardKeyWithTimestamp + " wantToRebuild=" + wantToRebuild + " }"; } @@ -477,8 +469,7 @@ public class NodeSpec { private Optional<URI> archiveUri = Optional.empty(); private Optional<ApplicationId> exclusiveTo = Optional.empty(); private List<TrustStoreItem> trustStore = List.of(); - private Optional<WireguardKey> wireguardPubkey = Optional.empty(); - private Optional<Instant> wireguardKeyTimestamp = Optional.empty(); + private Optional<WireguardKeyWithTimestamp> wireguardPubkey = Optional.empty(); private boolean wantToRebuild = false; public Builder() {} @@ -514,8 +505,7 @@ public class NodeSpec { node.archiveUri.ifPresent(this::archiveUri); node.exclusiveTo.ifPresent(this::exclusiveTo); trustStore(node.trustStore); - node.wireguardPubkey.ifPresent(this::wireguardPubkey); - node.wireguardKeyTimestamp.ifPresent(this::wireguardKeyTimestamp); + node.wireguardKeyWithTimestamp.ifPresent(this::wireguardKeyWithTimestamp); wantToRebuild(node.wantToRebuild); } @@ -704,13 +694,13 @@ public class NodeSpec { return this; } - public Builder wireguardPubkey(WireguardKey wireguardPubKey) { - this.wireguardPubkey = Optional.of(wireguardPubKey); + public Builder wireguardPubkey(WireguardKey wireguardPubkey) { + this.wireguardPubkey = Optional.of(new WireguardKeyWithTimestamp(wireguardPubkey, Instant.EPOCH)); return this; } - public Builder wireguardKeyTimestamp(Instant wireguardKeyTimestamp) { - this.wireguardKeyTimestamp = Optional.of(wireguardKeyTimestamp); + public Builder wireguardKeyWithTimestamp(WireguardKeyWithTimestamp wireguardPubKey) { + this.wireguardPubkey = Optional.of(wireguardPubKey); return this; } @@ -846,7 +836,7 @@ public class NodeSpec { wantedFirmwareCheck, currentFirmwareCheck, modelName, resources, realResources, ipAddresses, additionalIpAddresses, reports, events, parentHostname, archiveUri, exclusiveTo, trustStore, - wireguardPubkey, wireguardKeyTimestamp, wantToRebuild); + wireguardPubkey, wantToRebuild); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index a9cc2d698e9..17d3b51398f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; @@ -139,26 +140,28 @@ public class RealNodeRepository implements NodeRepository { return response.nodes.stream() .mapMulti((NodeRepositoryNode node, Consumer<WireguardPeer> consumer) -> { - if (node.wireguardPubkey == null || node.wireguardPubkey.isEmpty()) return; - List<VersionedIpAddress> ipAddresses = node.ipAddresses.stream() - .map(InetAddresses::forString) - .filter(address -> !address.isLoopbackAddress() && !address.isLinkLocalAddress() && !address.isSiteLocalAddress()) - .map(VersionedIpAddress::from) - .toList(); - if (ipAddresses.isEmpty()) return; + var keyWithTimestamp = createWireguardKeyWithTimestamp(node.wireguardKeyWithTimestamp, + node.wireguardPubkey, + node.wireguardKeyTimestamp); + if (keyWithTimestamp == null) return; - // Unbox to prevent NPE - long keyTimestamp = node.wireguardKeyTimestamp == null ? 0L : node.wireguardKeyTimestamp; + List<VersionedIpAddress> ipAddresses = getIpAddresses(node); + if (ipAddresses.isEmpty()) return; - consumer.accept(new WireguardPeer(HostName.of(node.hostname), - ipAddresses, - WireguardKey.from(node.wireguardPubkey), - Instant.ofEpochMilli(keyTimestamp))); + consumer.accept(new WireguardPeer(HostName.of(node.hostname), ipAddresses, keyWithTimestamp)); }) .sorted() .toList(); } + private static List<VersionedIpAddress> getIpAddresses(NodeRepositoryNode node) { + return node.ipAddresses.stream() + .map(InetAddresses::forString) + .filter(address -> !address.isLoopbackAddress() && !address.isLinkLocalAddress() && !address.isSiteLocalAddress()) + .map(VersionedIpAddress::from) + .toList(); + } + @Override public List<WireguardPeer> getConfigserverPeers() { GetWireguardResponse response = configServerApi.get("/nodes/v2/wireguard", GetWireguardResponse.class); @@ -246,8 +249,9 @@ public class RealNodeRepository implements NodeRepository { Optional.ofNullable(node.archiveUri).map(URI::create), Optional.ofNullable(node.exclusiveTo).map(ApplicationId::fromSerializedForm), trustStore, - Optional.ofNullable(node.wireguardPubkey).map(WireguardKey::from), - Optional.ofNullable(node.wireguardKeyTimestamp).map(Instant::ofEpochMilli), + Optional.ofNullable(createWireguardKeyWithTimestamp(node.wireguardKeyWithTimestamp, + node.wireguardPubkey, + node.wireguardKeyTimestamp)), node.wantToRebuild); } @@ -364,20 +368,39 @@ public class RealNodeRepository implements NodeRepository { node.trustStore = nodeAttributes.getTrustStore().stream() .map(item -> new NodeRepositoryNode.TrustStoreItem(item.fingerprint(), item.expiry().toEpochMilli())) .toList(); - node.wireguardPubkey = nodeAttributes.getWireguardPubkey().map(WireguardKey::value).orElse(null); + // This is used for patching, and timestamp must only be set on the server side, hence sending EPOCH. + node.wireguardKeyWithTimestamp = nodeAttributes.getWireguardPubkey() + .map(key -> new NodeRepositoryNode.WireguardKeyWithTimestamp(key.value(), 0L)) + .orElse(null); Map<String, JsonNode> reports = nodeAttributes.getReports(); node.reports = reports == null || reports.isEmpty() ? null : new TreeMap<>(reports); + // TODO wg: remove when all nodes are using new key+timestamp format + node.wireguardPubkey = nodeAttributes.getWireguardPubkey().map(WireguardKey::value).orElse(null); return node; } private static WireguardPeer createConfigserverPeer(GetWireguardResponse.Configserver configServer) { - // Unbox to prevent NPE - long keyTimestamp = configServer.wireguardKeyTimestamp == null ? 0L : configServer.wireguardKeyTimestamp; - return new WireguardPeer(HostName.of(configServer.hostname), configServer.ipAddresses.stream().map(VersionedIpAddress::from).toList(), - WireguardKey.from(configServer.wireguardPubkey), - Instant.ofEpochMilli(keyTimestamp)); + createWireguardKeyWithTimestamp(configServer.wireguardKeyWithTimestamp, + configServer.wireguardPubkey, + configServer.wireguardKeyTimestamp)); + } + + private static WireguardKeyWithTimestamp createWireguardKeyWithTimestamp(NodeRepositoryNode.WireguardKeyWithTimestamp wirguardJson, + String oldKeyJson, Long oldTimestampJson) { + if (wirguardJson != null && wirguardJson.key != null && ! wirguardJson.key.isEmpty()) { + return new WireguardKeyWithTimestamp(WireguardKey.from(wirguardJson.key), + Instant.ofEpochMilli(wirguardJson.timestamp)); + // TODO wg: remove when all nodes are using new key+timestamp format + } else if (oldKeyJson != null) { + var timestamp = oldTimestampJson != null ? oldTimestampJson : 0L; + return new WireguardKeyWithTimestamp(WireguardKey.from(oldKeyJson), + Instant.ofEpochMilli(timestamp)); + // TODO END + } else return null; + } + } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java index dcbf4cc163f..47903795ef7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java @@ -27,27 +27,23 @@ public class GetWireguardResponse { public static class Configserver { @JsonProperty("hostname") - public final String hostname; + public String hostname; @JsonProperty("ipAddresses") - public final List<String> ipAddresses; + public List<String> ipAddresses; + + @JsonProperty("wireguard") + public NodeRepositoryNode.WireguardKeyWithTimestamp wireguardKeyWithTimestamp; - @JsonProperty("wireguardPubkey") - public final String wireguardPubkey; + // TODO wg: remove when all nodes use new key+timestamp format + @JsonProperty("wireguardPubkey") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public String wireguardPubkey; @JsonProperty("wireguardKeyTimestamp") - public final Long wireguardKeyTimestamp; - - @JsonCreator - public Configserver(@JsonProperty("hostname") String hostname, - @JsonProperty("ipAddresses") List<String> ipAddresses, - @JsonProperty("wireguardPubkey") String wireguardPubkey, - @JsonProperty("wireguardKeyTimestamp") Long wireguardKeyTimestamp) { - this.hostname = hostname; - this.ipAddresses = ipAddresses; - this.wireguardPubkey = wireguardPubkey; - this.wireguardKeyTimestamp = wireguardKeyTimestamp; - } + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public Long wireguardKeyTimestamp; + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java index 3d0d052a877..35ca757ebbe 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java @@ -92,6 +92,10 @@ public class NodeRepositoryNode { @JsonProperty("trustStore") @JsonInclude(JsonInclude.Include.NON_EMPTY) public List<TrustStoreItem> trustStore; + @JsonProperty("wireguard") + public WireguardKeyWithTimestamp wireguardKeyWithTimestamp; + + // TODO wg: remove separate key and timestamp when all nodes use new keyWithTimestamp @JsonProperty("wireguardPubkey") @JsonInclude(JsonInclude.Include.NON_EMPTY) public String wireguardPubkey; @@ -141,13 +145,25 @@ public class NodeRepositoryNode { ", exclusiveTo='" + exclusiveTo + '\'' + ", history=" + history + ", trustStore=" + trustStore + - ", wireguardPubkey=" + wireguardPubkey + - ", wireguardKeyTimestamp=" + wireguardKeyTimestamp + + ", wireguard=" + wireguardKeyTimestamp + ", reports=" + reports + '}'; } @JsonIgnoreProperties(ignoreUnknown = true) + public static class WireguardKeyWithTimestamp { + @JsonProperty("key") + public String key; + @JsonProperty("timestamp") + public long timestamp; + + public WireguardKeyWithTimestamp(@JsonProperty("key") String key, @JsonProperty("timestamp") long timestamp) { + this.key = key; + this.timestamp = timestamp; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) public static class Owner { @JsonProperty("tenant") public String tenant; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java index b5428f57f08..e5ab9a1ce31 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java @@ -1,10 +1,9 @@ package com.yahoo.vespa.hosted.node.admin.wireguard; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; -import java.time.Instant; import java.util.List; /** @@ -15,8 +14,7 @@ import java.util.List; */ public record WireguardPeer(HostName hostname, List<VersionedIpAddress> ipAddresses, - WireguardKey publicKey, - Instant wireguardKeyTimestamp) implements Comparable<WireguardPeer> { + WireguardKeyWithTimestamp keyWithTimestamp) implements Comparable<WireguardPeer> { public WireguardPeer { if (ipAddresses.isEmpty()) throw new IllegalArgumentException("No IP addresses for peer node " + hostname.value()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 98e65d03f2f..ee3eac22d02 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; @@ -140,6 +141,7 @@ public class RealNodeRepositoryTest { var dockerImage = "registry.example.com/repo/image-1:6.2.3"; var wireguardKey = WireguardKey.from("111122223333444455556666777788889999000042c="); var wireguardKeyTimestamp = Instant.ofEpochMilli(123L); // Instant from clock in MockNodeRepository + var keyWithTimestamp = new WireguardKeyWithTimestamp(wireguardKey, wireguardKeyTimestamp); nodeRepositoryApi.updateNodeAttributes( hostname, @@ -151,8 +153,7 @@ public class RealNodeRepositoryTest { NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode(hostname).orElseThrow(); assertEquals(1, hostSpec.currentRestartGeneration().orElseThrow()); assertEquals(dockerImage, hostSpec.currentDockerImage().orElseThrow().asString()); - assertEquals(wireguardKey.value(), hostSpec.wireguardPubkey().orElseThrow().value()); - assertEquals(wireguardKeyTimestamp, hostSpec.wireguardKeyTimestamp().orElseThrow()); + assertEquals(keyWithTimestamp, hostSpec.wireguardKeyWithTimestamp().orElseThrow()); } @Test @@ -215,7 +216,7 @@ public class RealNodeRepositoryTest { assertWireguardPeer(cfgPeers.get(0), "cfg1.yahoo.com", "::201:1", "lololololololololololololololololololololoo=", - Instant.ofEpochMilli(456L)); + 456L); //// Exclave nodes //// @@ -227,16 +228,17 @@ public class RealNodeRepositoryTest { assertWireguardPeer(exclavePeers.get(0), "dockerhost2.yahoo.com", "::101:1", "000011112222333344445555666677778888999900c=", - Instant.ofEpochMilli(123L)); + 123L); } private void assertWireguardPeer(WireguardPeer peer, String hostname, String ipv6, - String publicKey, Instant keyTimestamp) { + String publicKey, long keyTimestamp) { assertEquals(hostname, peer.hostname().value()); assertEquals(1, peer.ipAddresses().size()); assertIp(peer.ipAddresses().get(0), ipv6, 6); - assertEquals(publicKey, peer.publicKey().value()); - assertEquals(keyTimestamp, peer.wireguardKeyTimestamp()); + var expectedKeyWithTimestamp = new WireguardKeyWithTimestamp(WireguardKey.from(publicKey), + Instant.ofEpochMilli(keyTimestamp)); + assertEquals(expectedKeyWithTimestamp, peer.keyWithTimestamp()); } private void assertIp(VersionedIpAddress ip, String expectedIp, int expectedVersion) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java index cd76b221c9e..6ee896e3db6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.wireguard; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; import org.junit.jupiter.api.Test; @@ -31,6 +32,7 @@ public class WireguardPeerTest { private static WireguardPeer peer(String hostname) { return new WireguardPeer(HostName.of(hostname), List.of(VersionedIpAddress.from("::1:1")), - WireguardKey.generateRandomForTesting(), Instant.EPOCH); + new WireguardKeyWithTimestamp(WireguardKey.generateRandomForTesting(), Instant.EPOCH)); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 24159b88a9b..d5e891a33c7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.lb.LoadBalancers; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -64,8 +64,7 @@ public final class Node implements Nodelike { private final CloudAccount cloudAccount; /** Only set for configservers and exclave nodes */ - private final Optional<WireguardKey> wireguardPubKey; - private final Optional<Instant> wireguardKeyTimestamp; + private final Optional<WireguardKeyWithTimestamp> wireguardPubKey; /** Record of the last event of each type happening to this node */ private final History history; @@ -96,8 +95,8 @@ public final class Node implements Nodelike { NodeType type, Reports reports, Optional<String> modelName, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveToApplicationId, Optional<Duration> hostTTL, Optional<Instant> hostEmptyAt, Optional<ClusterSpec.Type> exclusiveToClusterType, Optional<String> switchHostname, - List<TrustStoreItem> trustStoreItems, CloudAccount cloudAccount, Optional<WireguardKey> wireguardPubKey, - Optional<Instant> wireguardKeyTimestamp) { + List<TrustStoreItem> trustStoreItems, CloudAccount cloudAccount, + Optional<WireguardKeyWithTimestamp> wireguardPubKey) { this.id = Objects.requireNonNull(id, "A node must have an ID"); this.extraId = Objects.requireNonNull(extraId, "Extra ID cannot be null"); this.hostname = requireNonEmptyString(hostname, "A node must have a hostname"); @@ -120,7 +119,6 @@ public final class Node implements Nodelike { this.trustStoreItems = Objects.requireNonNull(trustStoreItems).stream().distinct().toList(); this.cloudAccount = Objects.requireNonNull(cloudAccount); this.wireguardPubKey = Objects.requireNonNull(wireguardPubKey); - this.wireguardKeyTimestamp = Objects.requireNonNull(wireguardKeyTimestamp); if (state == State.active) requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); @@ -264,15 +262,10 @@ public final class Node implements Nodelike { } /** Returns the wireguard public key of this node. Only relevant for enclave nodes. */ - public Optional<WireguardKey> wireguardPubKey() { + public Optional<WireguardKeyWithTimestamp> wireguardPubKey() { return wireguardPubKey; } - /** Returns the timestamp of the wireguard key of this node. Only relevant for enclave nodes. */ - public Optional<Instant> wireguardKeyTimestamp() { - return wireguardKeyTimestamp; - } - /** * Returns a copy of this where wantToFail is set to true and history is updated to reflect this. */ @@ -367,16 +360,14 @@ public final class Node implements Nodelike { public Node with(Status status) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a node with the type assigned to the given value */ public Node with(NodeType type) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a node with the flavor assigned to the given value */ @@ -385,40 +376,35 @@ public final class Node implements Nodelike { History updateHistory = history.with(new History.Event(History.Event.Type.resized, agent, instant)); return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, updateHistory, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with the reboot generation set to generation */ public Node withReboot(Generation generation) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with given id set */ public Node withId(String id) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with model name set to given value */ public Node withModelName(String modelName) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, Optional.of(modelName), reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with model name cleared */ public Node withoutModelName() { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, Optional.empty(), reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with a history record saying it was detected to be down at this instant */ @@ -460,24 +446,21 @@ public final class Node implements Nodelike { public Node with(Allocation allocation) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, Optional.of(allocation), history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node with IP config set to the given value. */ public Node with(IP.Config ipConfig) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node withParentHostname(String parentHostname) { return new Node(id, extraId, ipConfig, hostname, Optional.of(parentHostname), flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withReservedTo(TenantName tenant) { @@ -485,73 +468,59 @@ public final class Node implements Nodelike { throw new IllegalArgumentException("Only host nodes can be reserved, " + hostname + " has type " + type); return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, Optional.of(tenant), exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node which is not reserved to a tenant */ public Node withoutReservedTo() { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, Optional.empty(), exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExclusiveToApplicationId(ApplicationId exclusiveTo) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExtraId(Optional<String> extraId) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withHostTTL(Duration hostTTL) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, Optional.ofNullable(hostTTL), hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withHostEmptyAt(Instant hostEmptyAt) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, Optional.ofNullable(hostEmptyAt), - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExclusiveToClusterType(ClusterSpec.Type exclusiveTo) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - Optional.ofNullable(exclusiveTo), switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + Optional.ofNullable(exclusiveTo), switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } - public Node withWireguardPubkey(WireguardKey wireguardPubkey) { + public Node withWireguardPubkey(WireguardKeyWithTimestamp wireguardPubkey) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, Optional.ofNullable(wireguardPubkey), - wireguardKeyTimestamp); - } - - public Node withWireguardKeyTimestamp(Instant wireguardKeyTimestamp) { - return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - Optional.ofNullable(wireguardKeyTimestamp)); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, + Optional.ofNullable(wireguardPubkey)); } /** Returns a copy of this node with switch hostname set to given value */ public Node withSwitchHostname(String switchHostname) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, Optional.ofNullable(switchHostname), trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, Optional.ofNullable(switchHostname), trustStoreItems, cloudAccount, + wireguardPubKey); } /** Returns a copy of this node with switch hostname unset */ @@ -604,22 +573,19 @@ public final class Node implements Nodelike { public Node with(History history) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node with(Reports reports) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node with(List<TrustStoreItem> trustStoreItems) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } private static Optional<String> requireNonEmptyString(Optional<String> value, String message) { @@ -767,8 +733,7 @@ public final class Node implements Nodelike { private History history; private List<TrustStoreItem> trustStoreItems; private CloudAccount cloudAccount = CloudAccount.empty; - private WireguardKey wireguardPubKey; - private Instant wireguardKeyTimestamp; + private WireguardKeyWithTimestamp wireguardPubKey; private Builder(String id, String hostname, Flavor flavor, State state, NodeType type) { this.id = id; @@ -858,16 +823,11 @@ public final class Node implements Nodelike { return this; } - public Builder wireguardPubKey(WireguardKey wireguardPubKey) { + public Builder wireguardKey(WireguardKeyWithTimestamp wireguardPubKey) { this.wireguardPubKey = wireguardPubKey; return this; } - public Builder wireguardKeyTimestamp(Instant wireguardKeyTimestamp) { - this.wireguardKeyTimestamp = wireguardKeyTimestamp; - return this; - } - public Node build() { return new Node(id, Optional.empty(), Optional.ofNullable(ipConfig).orElse(IP.Config.EMPTY), hostname, Optional.ofNullable(parentHostname), flavor, Optional.ofNullable(status).orElseGet(Status::initial), state, Optional.ofNullable(allocation), @@ -875,7 +835,7 @@ public final class Node implements Nodelike { Optional.ofNullable(modelName), Optional.ofNullable(reservedTo), Optional.ofNullable(exclusiveToApplicationId), Optional.ofNullable(hostTTL), Optional.ofNullable(hostEmptyAt), Optional.ofNullable(exclusiveToClusterType), Optional.ofNullable(switchHostname), Optional.ofNullable(trustStoreItems).orElseGet(List::of), cloudAccount, - Optional.ofNullable(wireguardPubKey), Optional.ofNullable(wireguardKeyTimestamp)); + Optional.ofNullable(wireguardPubKey)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java index 8069c9c089b..286ec2451f8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java @@ -160,7 +160,7 @@ public class AllocatableResources { for (Node node : nodes) { sum = sum.add(nodeRepository.resourcesCalculator().realResourcesOf(node, nodeRepository).justNumbers()); } - return nodes.get(0).allocation().get().requestedResources().justNonNumbers() + return nodes.get(0).allocation().get().requestedResources() .withVcpu(sum.vcpu() / nodes.size()) .withMemoryGb(sum.memoryGb() / nodes.size()) .withDiskGb(sum.diskGb() / nodes.size()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java index e228d31384c..f42d1ce9bd3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java @@ -21,7 +21,8 @@ import java.util.Set; public class LoadBalancerInstance { private final Optional<DomainName> hostname; - private final Optional<String> ipAddress; + private final Optional<String> ip4Address; + private final Optional<String> ip6Address; private final Optional<DnsZone> dnsZone; private final Set<Integer> ports; private final Set<String> networks; @@ -30,11 +31,12 @@ public class LoadBalancerInstance { private final List<PrivateServiceId> serviceIds; private final CloudAccount cloudAccount; - public LoadBalancerInstance(Optional<DomainName> hostname, Optional<String> ipAddress, + public LoadBalancerInstance(Optional<DomainName> hostname, Optional<String> ip4Address, Optional<String> ip6Address, Optional<DnsZone> dnsZone, Set<Integer> ports, Set<String> networks, Set<Real> reals, ZoneEndpoint settings, List<PrivateServiceId> serviceIds, CloudAccount cloudAccount) { this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); - this.ipAddress = Objects.requireNonNull(ipAddress, "ip must be non-null"); + this.ip4Address = Objects.requireNonNull(ip4Address, "ip4Address must be non-null"); + this.ip6Address = Objects.requireNonNull(ip6Address, "ip6Address must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); this.ports = ImmutableSortedSet.copyOf(requirePorts(ports)); this.networks = ImmutableSortedSet.copyOf(Objects.requireNonNull(networks, "networks must be non-null")); @@ -43,9 +45,9 @@ public class LoadBalancerInstance { this.serviceIds = List.copyOf(Objects.requireNonNull(serviceIds, "private service id must be non-null")); this.cloudAccount = Objects.requireNonNull(cloudAccount, "cloudAccount must be non-null"); - if (hostname.isEmpty() == ipAddress.isEmpty()) { - throw new IllegalArgumentException("Exactly 1 of hostname=%s and ipAddress=%s must be set".formatted( - hostname.map(DomainName::value).orElse("<empty>"), ipAddress.orElse("<empty>"))); + if (hostname.isEmpty() == ip4Address.isEmpty()) { + throw new IllegalArgumentException("Exactly 1 of hostname=%s and ip4Address=%s must be set".formatted( + hostname.map(DomainName::value).orElse("<empty>"), ip4Address.orElse("<empty>"))); } } @@ -54,9 +56,14 @@ public class LoadBalancerInstance { return hostname; } - /** IP address of this (public) load balancer */ - public Optional<String> ipAddress() { - return ipAddress; + /** IPv4 address of this (public) load balancer */ + public Optional<String> ip4Address() { + return ip4Address; + } + + /** IPv6 address of this (public) load balancer */ + public Optional<String> ip6Address() { + return ip6Address; } /** ID of the DNS zone associated with this */ @@ -114,7 +121,7 @@ public class LoadBalancerInstance { public LoadBalancerInstance with(Set<Real> reals, ZoneEndpoint settings, Optional<PrivateServiceId> serviceId) { List<PrivateServiceId> ids = new ArrayList<>(serviceIds); serviceId.filter(id -> ! ids.contains(id)).ifPresent(ids::add); - return new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, + return new LoadBalancerInstance(hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, ids, cloudAccount); } @@ -123,7 +130,7 @@ public class LoadBalancerInstance { public LoadBalancerInstance withServiceIds(List<PrivateServiceId> serviceIds) { List<PrivateServiceId> ids = new ArrayList<>(serviceIds); for (PrivateServiceId id : this.serviceIds) if ( ! ids.contains(id)) ids.add(id); - return new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, settings, ids, cloudAccount); + return new LoadBalancerInstance(hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, ids, cloudAccount); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index a79766a577d..c79ccc2aece 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -57,6 +57,7 @@ public class LoadBalancerServiceMock implements LoadBalancerService { var instance = new LoadBalancerInstance( Optional.of(DomainName.of("lb-" + spec.application().toShortString() + "-" + spec.cluster().value())), Optional.empty(), + Optional.empty(), Optional.of(new DnsZone("zone-id-1")), Collections.singleton(4443), ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index e49d1b302cf..073662b39fe 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -45,6 +45,7 @@ public class SharedLoadBalancerService implements LoadBalancerService { return new LoadBalancerInstance(Optional.of(DomainName.of(vipHostname)), Optional.empty(), Optional.empty(), + Optional.empty(), Set.of(4443), Set.of(), spec.reals(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index 3c3868bfeb8..e4e08e5a15c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -47,6 +47,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; import static com.yahoo.stream.CustomCollectors.toLinkedMap; import static java.util.stream.Collectors.collectingAndThen; @@ -222,7 +223,7 @@ public class CuratorDb { node.type(), node.reports(), node.modelName(), node.reservedTo(), node.exclusiveToApplicationId(), node.hostTTL(), node.hostEmptyAt(), node.exclusiveToClusterType(), node.switchHostname(), node.trustedCertificates(), - node.cloudAccount(), node.wireguardPubKey(), node.wireguardKeyTimestamp()); + node.cloudAccount(), node.wireguardPubKey()); curatorTransaction.add(createOrSet(nodePath(newNode), nodeSerializer.toJson(newNode))); writtenNodes.add(newNode); } @@ -456,7 +457,12 @@ public class CuratorDb { transaction.onCommitted(() -> { for (var lb : loadBalancers) { if (lb.state() == fromState) continue; - Optional<String> target = lb.instance().flatMap(instance -> instance.hostname().map(DomainName::value).or(instance::ipAddress)); + Optional<String> target = lb.instance() + .flatMap(instance -> instance.hostname() + .map(DomainName::value) + .or(() -> Optional.of(Stream.concat(instance.ip4Address().stream(), + instance.ip6Address().stream()) + .collect(Collectors.joining(","))))); if (fromState == null) { log.log(Level.INFO, () -> "Creating " + lb.id() + target.map(t -> " (" + t + ")").orElse("") + " in " + lb.state()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index b85d96c6b54..d329676f842 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -45,6 +45,7 @@ public class LoadBalancerSerializer { private static final String idField = "id"; private static final String hostnameField = "hostname"; private static final String lbIpAddressField = "ipAddress"; + private static final String lbIp6AddressField = "ip6Address"; private static final String stateField = "state"; private static final String changedAtField = "changedAt"; private static final String dnsZoneField = "dnsZone"; @@ -69,7 +70,8 @@ public class LoadBalancerSerializer { root.setString(idField, loadBalancer.id().serializedForm()); loadBalancer.instance().flatMap(LoadBalancerInstance::hostname).ifPresent(hostname -> root.setString(hostnameField, hostname.value())); - loadBalancer.instance().flatMap(LoadBalancerInstance::ipAddress).ifPresent(ip -> root.setString(lbIpAddressField, ip)); + loadBalancer.instance().flatMap(LoadBalancerInstance::ip4Address).ifPresent(ip -> root.setString(lbIpAddressField, ip)); + loadBalancer.instance().flatMap(LoadBalancerInstance::ip6Address).ifPresent(ip -> root.setString(lbIp6AddressField, ip)); root.setString(stateField, asString(loadBalancer.state())); root.setLong(changedAtField, loadBalancer.changedAt().toEpochMilli()); loadBalancer.instance().flatMap(LoadBalancerInstance::dnsZone).ifPresent(dnsZone -> root.setString(dnsZoneField, dnsZone.id())); @@ -123,7 +125,8 @@ public class LoadBalancerSerializer { object.field(networksField).traverse((ArrayTraverser) (i, network) -> networks.add(network.asString())); Optional<DomainName> hostname = optionalString(object.field(hostnameField), Function.identity()).filter(s -> !s.isEmpty()).map(DomainName::of); - Optional<String> ipAddress = optionalString(object.field(lbIpAddressField), Function.identity()).filter(s -> !s.isEmpty()); + Optional<String> ip4Address = optionalString(object.field(lbIpAddressField), Function.identity()).filter(s -> !s.isEmpty()); + Optional<String> ip6Address = optionalString(object.field(lbIp6AddressField), Function.identity()).filter(s -> !s.isEmpty()); Optional<DnsZone> dnsZone = optionalString(object.field(dnsZoneField), DnsZone::new); ZoneEndpoint settings = zoneEndpoint(object.field(settingsField)); Optional<PrivateServiceId> serviceId = optionalString(object.field(serviceIdField), PrivateServiceId::of); @@ -131,9 +134,9 @@ public class LoadBalancerSerializer { object.field(serviceIdsField).traverse((ArrayTraverser) (__, serviceIdObject) -> serviceIds.add(PrivateServiceId.of(serviceIdObject.asString()))); if (serviceIds.isEmpty()) serviceId.ifPresent(serviceIds::add); // TODO: remove after winter vacation '23 CloudAccount cloudAccount = optionalString(object.field(cloudAccountField), CloudAccount::from).orElse(CloudAccount.empty); - Optional<LoadBalancerInstance> instance = hostname.isEmpty() && ipAddress.isEmpty() + Optional<LoadBalancerInstance> instance = hostname.isEmpty() && ip4Address.isEmpty() && ip6Address.isEmpty() ? Optional.empty() - : Optional.of(new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, settings, serviceIds, cloudAccount)); + : Optional.of(new LoadBalancerInstance(hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, serviceIds, cloudAccount)); return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()), instance, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 870e678a250..73531d650d5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.slime.ArrayTraverser; @@ -188,8 +189,10 @@ public class NodeSerializer { if (!node.cloudAccount().isUnspecified()) { object.setString(cloudAccountKey, node.cloudAccount().value()); } - node.wireguardPubKey().ifPresent(pubKey -> object.setString(wireguardPubKeyKey, pubKey.value())); - node.wireguardKeyTimestamp().ifPresent(timestamp -> object.setLong(wireguardKeyTimestampKey, timestamp.toEpochMilli())); + node.wireguardPubKey().ifPresent(pubKey -> { + object.setString(wireguardPubKeyKey, pubKey.key().value()); + object.setLong(wireguardKeyTimestampKey, pubKey.timestamp().toEpochMilli()); + }); } private void toSlime(Flavor flavor, Cursor object) { @@ -284,8 +287,7 @@ public class NodeSerializer { SlimeUtils.optionalString(object.field(switchHostnameKey)), trustedCertificatesFromSlime(object), SlimeUtils.optionalString(object.field(cloudAccountKey)).map(CloudAccount::from).orElse(CloudAccount.empty), - SlimeUtils.optionalString(object.field(wireguardPubKeyKey)).map(WireguardKey::from), - SlimeUtils.optionalInstant(object.field(wireguardKeyTimestampKey))); + wireguardKeyWithTimestampFromSlime(object.field(wireguardPubKeyKey), object.field(wireguardKeyTimestampKey))); } private Status statusFromSlime(Inspector object) { @@ -397,6 +399,13 @@ public class NodeSerializer { .toList(); } + private Optional<WireguardKeyWithTimestamp> wireguardKeyWithTimestampFromSlime(Inspector keyObject, Inspector timestampObject) { + if ( ! keyObject.valid()) return Optional.empty(); + return SlimeUtils.optionalString(keyObject).map( + key -> new WireguardKeyWithTimestamp(WireguardKey.from(key), + SlimeUtils.optionalInstant(timestampObject).orElse(null))); + } + // ----------------- Enum <-> string mappings ---------------------------------------- /** Returns the event type, or null if this event type should be ignored */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index 09f947503f6..20aa7d8181e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -57,7 +57,8 @@ public class LoadBalancersResponse extends SlimeJsonResponse { lbObject.setString("instance", lb.id().application().instance().value()); lbObject.setString("cluster", lb.id().cluster().value()); lb.instance().flatMap(LoadBalancerInstance::hostname).ifPresent(hostname -> lbObject.setString("hostname", hostname.value())); - lb.instance().flatMap(LoadBalancerInstance::ipAddress).ifPresent(ipAddress -> lbObject.setString("ipAddress", ipAddress)); + lb.instance().flatMap(LoadBalancerInstance::ip4Address).ifPresent(ip -> lbObject.setString("ipAddress", ip)); + lb.instance().flatMap(LoadBalancerInstance::ip6Address).ifPresent(ip -> lbObject.setString("ip6Address", ip)); lb.instance().flatMap(LoadBalancerInstance::dnsZone).ifPresent(dnsZone -> lbObject.setString("dnsZone", dnsZone.id())); Cursor networkArray = lbObject.setArray("networks"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 9f1ab3dc3d5..cad034e01aa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; @@ -108,7 +109,8 @@ public class NodePatcher { "reports", "trustStore", "vespaVersion", - "wireguardPubkey")); + "wireguardPubkey", // TODO wg: remove when all nodes use new key+timestamp format + "wireguard")); if (!disallowedFields.isEmpty()) { throw new IllegalArgumentException("Patching fields not supported: " + disallowedFields); } @@ -271,9 +273,13 @@ public class NodePatcher { return value.type() == Type.NIX ? node.withoutSwitchHostname() : node.withSwitchHostname(value.asString()); case "trustStore": return nodeWithTrustStore(node, value); - case "wireguardPubkey": - return node.withWireguardPubkey(SlimeUtils.optionalString(value).map(WireguardKey::new).orElse(null)) - .withWireguardKeyTimestamp(clock.instant()); + case "wireguard": + // This is where we set the key timestamp. + var key = SlimeUtils.optionalString(value.field("key")).map(WireguardKey::new).orElse(null); + return node.withWireguardPubkey(new WireguardKeyWithTimestamp(key, clock.instant())); + case "wireguardPubkey": // TODO wg: remove when all nodes use new key+timestamp format + var oldKey = SlimeUtils.optionalString(value).map(WireguardKey::new).orElse(null); + return node.withWireguardPubkey(new WireguardKeyWithTimestamp(oldKey, clock.instant())); default: throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index a8f526544d7..05bb0a27d69 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.restapi.SlimeJsonResponse; @@ -192,8 +193,13 @@ class NodesResponse extends SlimeJsonResponse { if (!node.cloudAccount().isUnspecified()) { object.setString("cloudAccount", node.cloudAccount().value()); } - node.wireguardPubKey().ifPresent(key -> object.setString("wireguardPubkey", key.value())); - node.wireguardKeyTimestamp().ifPresent(timestamp -> object.setLong("wireguardKeyTimestamp", timestamp.toEpochMilli())); + node.wireguardPubKey().ifPresent(key -> toSlime(key, object.setObject("wireguard"))); + + // TODO wg: remove when all nodes have upgraded to new key+timestamp format + node.wireguardPubKey().ifPresent(key -> { + object.setString("wireguardPubkey", key.key().value()); + object.setLong("wireguardKeyTimestamp", key.timestamp().toEpochMilli()); + }); } private Version resolveVersionFlag(StringFlag flag, Node node, Allocation allocation) { @@ -237,6 +243,11 @@ class NodesResponse extends SlimeJsonResponse { } } + static void toSlime(WireguardKeyWithTimestamp keyWithTimestamp, Cursor object) { + object.setString("key", keyWithTimestamp.key().value()); + object.setLong("timestamp", keyWithTimestamp.timestamp().toEpochMilli()); + } + private Optional<DockerImage> currentContainerImage(Node node) { if (node.status().containerImage().isPresent()) { return node.status().containerImage(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java index 16e85dfa48a..e29c4f1b87a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java @@ -1,7 +1,7 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.hosted.provision.Node; @@ -10,9 +10,9 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.IP; import java.net.InetAddress; -import java.time.Instant; import java.util.List; -import java.util.Optional; + +import static com.yahoo.vespa.hosted.provision.restapi.NodesResponse.toSlime; /** * A response containing the wireguard peer config for each configserver that has a public key. @@ -36,17 +36,20 @@ public class WireguardResponse extends SlimeJsonResponse { .toList(); if (ipAddresses.isEmpty()) continue; - addConfigserver(cfgArray.addObject(), cfg.hostname(), cfg.wireguardPubKey().get(), - cfg.wireguardKeyTimestamp(), ipAddresses); + addConfigserver(cfgArray.addObject(), cfg.hostname(), cfg.wireguardPubKey().get(), ipAddresses); } } - private void addConfigserver(Cursor cfgEntry, String hostname, WireguardKey key, Optional<Instant> keyTimestamp, + private void addConfigserver(Cursor cfgEntry, String hostname, WireguardKeyWithTimestamp keyWithTimestamp, List<String> ipAddresses) { cfgEntry.setString("hostname", hostname); - cfgEntry.setString("wireguardPubkey", key.value()); - cfgEntry.setLong("wireguardKeyTimestamp", keyTimestamp.orElse(Instant.EPOCH).toEpochMilli()); + + // TODO wg: remove when all nodes are using new key+timestamp format + cfgEntry.setString("wireguardPubkey", keyWithTimestamp.key().value()); + cfgEntry.setLong("wireguardKeyTimestamp", keyWithTimestamp.timestamp().toEpochMilli()); + NodesResponse.ipAddressesToSlime(ipAddresses, cfgEntry.setArray("ipAddresses")); + toSlime(keyWithTimestamp, cfgEntry.setObject("wireguard")); } private static boolean isPublicIp(String ipAddress) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 5cdb08d6fc6..a72c2fb0b9c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -79,7 +79,7 @@ public class MockHostProvisioner implements HostProvisioner { if (hostFlavor == null) hostFlavor = flavors.stream() .filter(f -> request.sharing() == HostSharing.exclusive ? compatible(f, request.resources()) - : f.resources().satisfies(request.resources())) + : satisfies(f, request.resources())) .filter(f -> realHostResourcesWithinLimits.test(f.resources())) .findFirst() .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + request.resources(), true)); @@ -223,6 +223,10 @@ public class MockHostProvisioner implements HostProvisioner { return flavor.resources().compatibleWith(resourcesToVerify); } + public boolean satisfies(Flavor flavor, NodeResources resources) { + return flavor.resources().satisfies(resources); + } + private List<HostName> createHostnames(NodeType hostType, Flavor flavor, int hostIndex) { long numAddresses = Math.max(2, Math.round(flavor.resources().bandwidthGbps())); return IntStream.range(1, (int) numAddresses) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 72225763381..2fb549acc11 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -21,6 +21,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.ZoneEndpoint; import com.yahoo.config.provision.ZoneEndpoint.AccessType; @@ -161,8 +162,8 @@ public class MockNodeRepository extends NodeRepository { // Emulate host in tenant account nodes.add(Node.create("dockerhost2", ipConfig(101, 1, 3), "dockerhost2.yahoo.com", flavors.getFlavorOrThrow("large"), NodeType.host) - .wireguardPubKey(WireguardKey.from("000011112222333344445555666677778888999900c=")) - .wireguardKeyTimestamp(Instant.ofEpochMilli(123L)) + .wireguardKey(new WireguardKeyWithTimestamp(WireguardKey.from("000011112222333344445555666677778888999900c="), + Instant.ofEpochMilli(123L))) .cloudAccount(tenantAccount).build()); nodes.add(Node.create("dockerhost3", ipConfig(102, 1, 3), "dockerhost3.yahoo.com", flavors.getFlavorOrThrow("large"), NodeType.host).cloudAccount(defaultCloudAccount).build()); @@ -176,8 +177,8 @@ public class MockNodeRepository extends NodeRepository { // Config servers nodes.add(Node.create("cfg1", ipConfig(201), "cfg1.yahoo.com", flavors.getFlavorOrThrow("default"), NodeType.config) .cloudAccount(defaultCloudAccount) - .wireguardPubKey(WireguardKey.from("lololololololololololololololololololololoo=")) - .wireguardKeyTimestamp(Instant.ofEpochMilli(456L)) + .wireguardKey(new WireguardKeyWithTimestamp(WireguardKey.from("lololololololololololololololololololololoo="), + Instant.ofEpochMilli(456L))) .build()); nodes.add(Node.create("cfg2", ipConfig(202), "cfg2.yahoo.com", flavors.getFlavorOrThrow("default"), NodeType.config) .cloudAccount(defaultCloudAccount) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java index f64e50310bb..3f66b2c94d8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -25,6 +26,7 @@ import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.hosted.provision.maintenance.SwitchRebalancer; import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.persistence.ApplicationSerializer; import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import com.yahoo.vespa.hosted.provision.persistence.NodeSerializer; @@ -45,9 +47,10 @@ import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; +import java.util.Iterator; import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; +import java.util.Map; +import java.util.function.BiConsumer; import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.IntStream; @@ -75,8 +78,8 @@ public class RealDataScenarioTest { @Ignore @Test - public void test() { - ProvisioningTester tester = tester(SystemName.Public, CloudName.AWS, Environment.prod, parseFlavors(Path.of("/tmp/node-flavors.xml"))); + public void test() throws Exception { + ProvisioningTester tester = tester(SystemName.Public, CloudName.AWS, Environment.prod, CloudAccount.empty, parseFlavors(Path.of("/tmp/node-flavors.xml"))); initFromZk(tester.nodeRepository(), Path.of("/tmp/snapshot")); ApplicationId app = ApplicationId.from("tenant", "app", "default"); @@ -132,41 +135,34 @@ public class RealDataScenarioTest { } } - private static void initFromZk(NodeRepository nodeRepository, Path pathToZkSnapshot) { + private static void initFromZk(NodeRepository nodeRepository, Path pathToZkSnapshot) throws Exception { NodeSerializer nodeSerializer = new NodeSerializer(nodeRepository.flavors()); - AtomicBoolean nodeNext = new AtomicBoolean(false); - Pattern zkNodePathPattern = Pattern.compile(".?/provision/v1/nodes/[a-z0-9.-]+\\.(com|cloud).?"); - Consumer<String> consumer = input -> { - if (nodeNext.get()) { - String json = input.substring(input.indexOf("{\""), input.lastIndexOf('}') + 1); - Node node = nodeSerializer.fromJson(json.getBytes(UTF_8)); - nodeRepository.database().addNodesInState(new LockedNodeList(List.of(node), () -> { }), node.state(), Agent.system); - nodeNext.set(false); - } else { - if (!zkNodePathPattern.matcher(input).matches()) return; - if (nodeNext.getAndSet(true)) - throw new IllegalStateException("Expected to find node JSON, but found another node path: " + input); + Map<Pattern, BiConsumer<byte[], NestedTransaction>> jsonConsumerByPathPattern = Map.of( + Pattern.compile(".?/provision/v1/nodes/[a-z0-9.-]+\\.(com|cloud).?"), (json, transaction) -> { + Node node = nodeSerializer.fromJson(json); + nodeRepository.database().addNodesInState(new LockedNodeList(List.of(node), () -> { }), node.state(), Agent.system, transaction); + }, + Pattern.compile(".?/provision/v1/applications/[a-z0-9:-]+.?"), (json, transaction) -> + nodeRepository.database().writeApplication(ApplicationSerializer.fromJson(json), transaction)); + + try (StringsIterator iterator = new StringsIterator(pathToZkSnapshot); NestedTransaction transaction = new NestedTransaction()) { + while (iterator.hasNext()) { + String s1 = iterator.next(); + if (!iterator.hasNext()) break; + for (var entry : jsonConsumerByPathPattern.entrySet()) { + if (!entry.getKey().matcher(s1).matches()) continue; + String s2 = iterator.next(); + byte[] json = s2.substring(s2.indexOf("{\""), s2.lastIndexOf('}') + 1).getBytes(UTF_8); + entry.getValue().accept(json, transaction); + break; + } } - }; - - try (BufferedReader reader = new BufferedReader( - new InputStreamReader(Files.newInputStream(pathToZkSnapshot), UTF_8))) { - StringBuilder sb = new StringBuilder(1000); - for (int r; (r = reader.read()) != -1; ) { - if (r < 0x20 || r >= 0x7F) { - if (sb.length() > 0) { - consumer.accept(sb.toString()); - sb.setLength(0); - } - } else sb.append((char) r); - } - } catch (IOException e) { - throw new UncheckedIOException(e); + transaction.commit(); } } - private static ProvisioningTester tester(SystemName systemName, CloudName cloudName, Environment environment, List<Flavor> flavors) { - Cloud cloud = Cloud.builder().name(cloudName).dynamicProvisioning(cloudName != CloudName.YAHOO).build(); + private static ProvisioningTester tester(SystemName systemName, CloudName cloudName, Environment environment, CloudAccount cloudAccount, List<Flavor> flavors) { + Cloud cloud = Cloud.builder().name(cloudName).dynamicProvisioning(cloudName != CloudName.YAHOO).account(cloudAccount).build(); NameResolver nameResolver = cloudName == CloudName.YAHOO ? new DnsNameResolver() : new MockNameResolver().mockAnyLookup(); ProvisioningTester.Builder builder = new ProvisioningTester.Builder() .zone(new Zone(cloud, systemName, environment, RegionName.defaultName())) @@ -179,4 +175,42 @@ public class RealDataScenarioTest { return builder.build(); } + /** Extracts sequences longer than 5 printable characters from a binary file, similar to `strings` command */ + private static class StringsIterator implements Iterator<String>, AutoCloseable { + private final BufferedReader reader; + private final StringBuilder sb = new StringBuilder(1000); + private StringsIterator(Path path) throws IOException { + this.reader = new BufferedReader(new InputStreamReader(Files.newInputStream(path), UTF_8)); + } + + @Override + public boolean hasNext() { + if (!sb.isEmpty()) return true; + try { + for (int r; (r = reader.read()) != -1; ) { + if (r < 0x20 || r >= 0x7F) { + if (sb.isEmpty()) continue; // Still haven't encountered any real data + if (sb.length() > 5) break; // We (probably) found some real data + sb.setLength(0); // Probably some random binary data that happened to be a printable character, reset + } else sb.append((char) r); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return !sb.isEmpty(); + } + + @Override + public String next() { + String next = sb.toString(); + sb.setLength(0); + return next; + } + + @Override + public void close() throws Exception { + reader.close(); + } + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 52d4c85bcaf..aa8ff1245d7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -32,6 +32,25 @@ import static org.junit.Assert.assertTrue; public class AutoscalingTest { @Test + public void test_autoscaling_with_gpu() { + var resources = new NodeResources(8, 32, 225, 0.1, fast, StorageType.local, NodeResources.Architecture.x86_64, new NodeResources.GpuResources(1, 16)); + var min = new ClusterResources( 8, 1, resources); + var now = new ClusterResources(12, 1, resources); + var max = new ClusterResources(12, 1, resources); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .clusterType(ClusterSpec.Type.container) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); + fixture.tester.clock().advance(Duration.ofDays(2)); + fixture.loader().applyLoad(new Load(0.8f, 0.17, 0.12), 1, true, true, 100); + var result = fixture.autoscale(); + assertTrue(result.resources().isEmpty()); + assertEquals(Autoscaling.Status.insufficient, result.status()); + } + + @Test public void test_autoscaling_nodes_only() { var resources = new NodeResources(16, 32, 200, 0.1); var min = new ClusterResources( 8, 1, resources); @@ -112,7 +131,7 @@ public class AutoscalingTest { fixture.loader().applyLoad(new Load(0.1, 0.1, 0.1), 3); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 1); fixture.tester().assertResources("Scaling up since resource usage is too high", - 8, 1, 5.3, 17.5, 75.4, + 8, 1, 5.3, 17.0, 75.1, fixture.autoscale()); } @@ -173,7 +192,7 @@ public class AutoscalingTest { fixture.setScalingDuration(Duration.ofHours(12)); // Fixture sets last completion to be 1 day into the past fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10); fixture.tester().assertResources("Scaling up (only) since resource usage is too high", - 5, 1, 11.7, 15.4, 132.0, + 5, 1, 11.7, 14.9, 131.5, fixture.autoscale()); } @@ -185,7 +204,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10); fixture.tester().assertResources("Scaling cpu and disk up and memory down", - 5, 1, 11.7, 4.0, 132.0, + 5, 1, 11.7, 4.0, 131.5, fixture.autoscale()); } @@ -208,7 +227,7 @@ public class AutoscalingTest { fixture.loader().applyCpuLoad(0.70, 1); fixture.loader().applyCpuLoad(0.01, 100); fixture.tester().assertResources("Scaling up since peak resource usage is too high", - 5, 1, 7.1, 12.3, 50.7, + 5, 1, 7.1, 11.9, 50.5, fixture.autoscale()); } @@ -355,7 +374,7 @@ public class AutoscalingTest { fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.05f, 0.05f, 0.05f), 120); fixture.tester().assertResources("Scaling down to limit since resource usage is low", - 4, 1, 1.8, 7.4, 23.5, + 4, 1, 1.8, 7.4, 23.4, fixture.autoscale()); } @@ -459,7 +478,7 @@ public class AutoscalingTest { fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(1.0, 120); fixture.tester().assertResources("Suggesting above capacity limit", - 5, 1, 10.2, 12.3, 50.7, + 5, 1, 10.2, 11.9, 50.5, fixture.tester().suggest(fixture.applicationId, fixture.clusterSpec.id(), min, min)); } @@ -663,7 +682,7 @@ public class AutoscalingTest { fixture.tester().clock().advance(Duration.ofHours(12 * 3 + 1)); fixture.loader().applyCpuLoad(0.02, 5); fixture.tester().assertResources("Scaling down since enough time has passed", - 5, 1, 1.0, 12.3, 50.7, + 5, 1, 1.0, 11.9, 50.5, fixture.autoscale()); } @@ -707,7 +726,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.25, 200); fixture.tester().assertResources("Scale up since we assume we need 2x cpu for growth when no scaling time data", - 5, 1, 2.6, 12.3, 50.7, + 5, 1, 2.6, 11.9, 50.5, fixture.autoscale()); fixture.setScalingDuration(Duration.ofHours(8)); @@ -716,7 +735,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.20, 200); fixture.tester().assertResources("Scale down since observed growth is slower than scaling time", - 5, 1, 1.6, 12.3, 50.7, + 5, 1, 1.6, 11.9, 50.5, fixture.autoscale()); fixture.setScalingDuration(Duration.ofHours(8)); @@ -727,7 +746,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.25, 200); fixture.tester().assertResources("Scale up since observed growth is faster than scaling time", - 5, 1, 2.4, 12.3, 50.7, + 5, 1, 2.4, 11.9, 50.5, fixture.autoscale()); } @@ -744,7 +763,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.7, 200); fixture.tester().assertResources("Scale up slightly since observed growth is faster than scaling time, but we are not confident", - 5, 1, 2.2, 12.3, 50.7, + 5, 1, 2.2, 11.9, 50.5, fixture.autoscale()); } @@ -763,7 +782,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester.assertResources("Query and write load is equal -> scale up somewhat", - 5, 1, 2.9, 12.3, 50.7, + 5, 1, 2.9, 11.9, 50.5, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -772,7 +791,7 @@ public class AutoscalingTest { fixture.loader().addCpuMeasurements(0.4, 200); // TODO: Ackhually, we scale up less here - why? fixture.tester().assertResources("Query load is 4x write load -> scale up more", - 5, 1, 2.2, 12.3, 50.7, + 5, 1, 2.2, 11.9, 50.5, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -780,7 +799,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester().assertResources("Write load is 10x query load -> scale down", - 5, 1, 1.3, 12.3, 50.7, + 5, 1, 1.3, 11.9, 50.5, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -788,7 +807,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester().assertResources("Query only -> larger", - 5, 1, 3.5, 12.3, 50.7, + 5, 1, 3.5, 11.9, 50.5, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -796,7 +815,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester().assertResources("Write only -> smallest possible", - 5, 1, 1.0, 12.3, 50.7, + 5, 1, 1.0, 11.9, 50.5, fixture.autoscale()); } @@ -825,7 +844,7 @@ public class AutoscalingTest { fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200); fixture.tester().assertResources("Scale only to a single node and group since this is dev", - 1, 1, 0.1, 23.6, 105.6, + 1, 1, 0.1, 22.9, 105.2, fixture.autoscale()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java index be7bc3c44a8..f2b80adc513 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java @@ -32,7 +32,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 3.4, 7.4, 29.0, + 8, 1, 3.4, 7.2, 28.8, fixture.autoscale()); // Higher query rate @@ -40,7 +40,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 6.8, 7.4, 29.0, + 8, 1, 6.8, 7.2, 28.8, fixture.autoscale()); // Higher headroom @@ -48,7 +48,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 4.0, 7.4, 29.0, + 8, 1, 4.0, 7.2, 28.8, fixture.autoscale()); // Higher per query cost @@ -56,7 +56,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 5.1, 7.4, 29.0, + 8, 1, 5.1, 7.2, 28.8, fixture.autoscale()); // Bcp elsewhere is 0 - use local only @@ -64,7 +64,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(0, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling using local info", - 8, 1, 1, 7.4, 29.0, + 8, 1, 1, 7.2, 28.8, fixture.autoscale()); } @@ -85,7 +85,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 11.7, 43.2, 190.0, + 3, 3, 11.7, 41.8, 189.3, fixture.autoscale()); // Higher query rate @@ -93,7 +93,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 23.1, 43.2, 190.0, + 3, 3, 23.1, 41.8, 189.3, fixture.autoscale()); // Higher headroom @@ -101,7 +101,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 13.8, 43.2, 190.0, + 3, 3, 13.8, 41.8, 189.3, fixture.autoscale()); // Higher per query cost @@ -109,7 +109,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 17.4, 43.2, 190.0, + 3, 3, 17.4, 41.8, 189.3, fixture.autoscale()); } @@ -186,7 +186,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.3, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 11.9, 7.4, 29.0, + 8, 1, 11.9, 7.2, 28.8, fixture.autoscale()); // Some local traffic @@ -196,7 +196,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration1.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 10.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 6.8, 7.4, 29.0, + 8, 1, 6.8, 7.2, 28.8, fixture.autoscale()); // Enough local traffic to get half the votes @@ -206,7 +206,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration2.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 50.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 3.0, 7.4, 29.0, + 8, 1, 3.0, 7.2, 28.8, fixture.autoscale()); // Mostly local @@ -216,7 +216,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration3.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 90.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 2.2, 7.4, 29.0, + 8, 1, 2.2, 7.2, 28.8, fixture.autoscale()); // Local only @@ -226,7 +226,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration4.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 100.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 2.1, 7.4, 29.0, + 8, 1, 2.1, 7.2, 28.8, fixture.autoscale()); // No group info, should be the same as the above @@ -236,7 +236,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration5.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 100.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 2.1, 7.4, 29.0, + 8, 1, 2.1, 7.2, 28.8, fixture.autoscale()); // 40 query rate, no group info (for reference to the below) @@ -246,7 +246,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration6.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 40.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 1.5, 7.4, 29.0, + 8, 1, 1.5, 7.2, 28.8, fixture.autoscale()); // Local query rate is too low but global is even lower so disregard it, giving the same as above @@ -256,7 +256,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration7.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 40.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 1.5, 7.4, 29.0, + 8, 1, 1.5, 7.2, 28.8, fixture.autoscale()); // Local query rate is too low to be fully confident, and so is global but as it is slightly larger, incorporate it slightly @@ -266,7 +266,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration8.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 40.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 1.8, 7.4, 29.0, + 8, 1, 1.8, 7.2, 28.8, fixture.autoscale()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsNodeTypes.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsNodeTypes.java index df7c468f035..6225982af42 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsNodeTypes.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsNodeTypes.java @@ -11,11 +11,12 @@ import java.util.Optional; import static com.yahoo.config.provision.NodeResources.Architecture.arm64; import static com.yahoo.config.provision.NodeResources.Architecture.x86_64; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; +import static com.yahoo.config.provision.NodeResources.GpuResources; import static com.yahoo.config.provision.NodeResources.StorageType.local; import static com.yahoo.config.provision.NodeResources.StorageType.remote; /** - * Returns the information about all AWS node types supported on Vespa Cloud as of 2022-10-31. + * Returns the information about all AWS node types supported on Vespa Cloud as of 2023-09-28. * * @author bratseth */ @@ -45,13 +46,34 @@ public class AwsNodeTypes { new VespaFlavor("m5d_16xlarge", 64.0, 64.0, 256.0, 246.0, 2400.0, 10.0, fast, local, x86_64), new VespaFlavor("m5_24xlarge", 96.0, 96.0, 384.0, 370.0, 16384.0, 10.0, fast, remote, x86_64), new VespaFlavor("m5d_24xlarge", 96.0, 96.0, 384.0, 370.0, 3600.0, 10.0, fast, local, x86_64), + new VespaFlavor("m6i_large", 2.0, 2.0, 8.0, 7.3, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("m6id_large", 2.0, 2.0, 8.0, 7.3, 118.0, 10.0, fast, local, x86_64), + new VespaFlavor("m6i_xlarge", 4.0, 4.0, 16.0, 15.1, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("m6id_xlarge", 4.0, 4.0, 16.0, 15.1, 237.0, 10.0, fast, local, x86_64), + new VespaFlavor("m6i_2xlarge", 8.0, 8.0, 32.0, 30.5, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("m6id_2xlarge", 8.0, 8.0, 32.0, 30.5, 474.0, 10.0, fast, local, x86_64), + new VespaFlavor("m6i_4xlarge", 16.0, 16.0, 64.0, 61.3, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("m6id_4xlarge", 16.0, 16.0, 64.0, 61.3, 950.0, 10.0, fast, local, x86_64), + new VespaFlavor("m6i_8xlarge", 32.0, 32.0, 128.0, 123.0, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("m6id_8xlarge", 32.0, 32.0, 128.0, 123.0, 1900.0, 10.0, fast, local, x86_64), + new VespaFlavor("m6i_12xlarge", 48.0, 48.0, 192.0, 185.0, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("m6id_12xlarge", 48.0, 48.0, 192.0, 185.0, 2850.0, 10.0, fast, local, x86_64), + new VespaFlavor("m6i_16xlarge", 64.0, 64.0, 256.0, 246.0, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("m6id_16xlarge", 64.0, 64.0, 256.0, 246.0, 3800.0, 10.0, fast, local, x86_64), new VespaFlavor("m6g_large", 2.0, 2.0, 8.0, 7.3, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("m6gd_large", 2.0, 2.0, 8.0, 7.3, 118.0, 10.0, fast, local, arm64), new VespaFlavor("m6g_xlarge", 4.0, 4.0, 16.0, 15.1, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("m6gd_xlarge", 4.0, 4.0, 16.0, 15.1, 237.0, 10.0, fast, local, arm64), new VespaFlavor("m6g_2xlarge", 8.0, 8.0, 32.0, 30.5, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("m6gd_2xlarge", 8.0, 8.0, 32.0, 30.5, 474.0, 10.0, fast, local, arm64), new VespaFlavor("m6g_4xlarge", 16.0, 16.0, 64.0, 61.3, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("m6gd_4xlarge", 16.0, 16.0, 64.0, 61.3, 950.0, 10.0, fast, local, arm64), new VespaFlavor("m6g_8xlarge", 32.0, 32.0, 128.0, 123.0, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("m6gd_8xlarge", 32.0, 32.0, 128.0, 123.0, 1900.0, 10.0, fast, local, arm64), new VespaFlavor("m6g_12xlarge", 48.0, 48.0, 192.0, 185.0, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("m6gd_12xlarge", 48.0, 48.0, 192.0, 185.0, 2850.0, 10.0, fast, local, arm64), new VespaFlavor("m6g_16xlarge", 64.0, 64.0, 256.0, 246.0, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("m6gd_16xlarge", 64.0, 64.0, 256.0, 246.0, 3800.0, 10.0, fast, local, arm64), new VespaFlavor("c5_large", 2.0, 2.0, 4.0, 3.5, 16384.0, 10.0, fast, remote, x86_64), new VespaFlavor("c5d_large", 2.0, 2.0, 4.0, 3.5, 50.0, 10.0, fast, local, x86_64), new VespaFlavor("c5_xlarge", 4.0, 4.0, 8.0, 7.3, 16384.0, 10.0, fast, remote, x86_64), @@ -68,10 +90,44 @@ public class AwsNodeTypes { new VespaFlavor("c5d_18xlarge", 72.0, 72.0, 144.0, 137.0, 1800.0, 10.0, fast, local, x86_64), new VespaFlavor("c5_24xlarge", 96.0, 96.0, 192.0, 185.0, 16384.0, 10.0, fast, remote, x86_64), new VespaFlavor("c5d_24xlarge", 96.0, 96.0, 192.0, 185.0, 3600.0, 10.0, fast, local, x86_64), + new VespaFlavor("c6i_large", 2.0, 2.0, 4.0, 3.5, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("c6id_large", 2.0, 2.0, 4.0, 3.5, 118.0, 10.0, fast, local, x86_64), + new VespaFlavor("c6i_xlarge", 4.0, 4.0, 8.0, 7.4, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("c6id_xlarge", 4.0, 4.0, 8.0, 7.4, 237.0, 10.0, fast, local, x86_64), + new VespaFlavor("c6i_2xlarge", 8.0, 8.0, 16.0, 15.1, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("c6id_2xlarge", 8.0, 8.0, 16.0, 15.1, 474.0, 10.0, fast, local, x86_64), + new VespaFlavor("c6i_4xlarge", 16.0, 16.0, 32.0, 30.6, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("c6id_4xlarge", 16.0, 16.0, 32.0, 30.6, 950.0, 10.0, fast, local, x86_64), + new VespaFlavor("c6i_8xlarge", 32.0, 32.0, 64.0, 61.7, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("c6id_8xlarge", 32.0, 32.0, 64.0, 61.7, 1900.0, 10.0, fast, local, x86_64), + new VespaFlavor("c6i_12xlarge", 48.0, 48.0, 96.0, 92.6, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("c6id_12xlarge", 48.0, 48.0, 96.0, 92.6, 2850.0, 10.0, fast, local, x86_64), + new VespaFlavor("c6i_16xlarge", 64.0, 64.0, 128.0, 123.0, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("c6id_16xlarge", 64.0, 64.0, 128.0, 123.0, 3800.0, 10.0, fast, local, x86_64), + new VespaFlavor("c6i_24xlarge", 96.0, 96.0, 192.0, 185.6, 16384.0, 10.0, fast, remote, x86_64), + new VespaFlavor("c6id_24xlarge", 96.0, 96.0, 192.0, 185.6, 5700.0, 10.0, fast, local, x86_64), new VespaFlavor("c5ad_8xlarge", 32.0, 32.0, 64.0, 62.0, 1200.0, 10.0, fast, local, x86_64), + new VespaFlavor("c6g_large", 2.0, 2.0, 4.0, 3.35, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("c6gd_large", 2.0, 2.0, 4.0, 3.35, 118.0, 10.0, fast, local, arm64), + new VespaFlavor("c6g_xlarge", 4.0, 4.0, 8.0, 7.3, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("c6gd_xlarge", 4.0, 4.0, 8.0, 7.3, 237.0, 10.0, fast, local, arm64), + new VespaFlavor("c6g_2xlarge", 8.0, 8.0, 16.0, 15.1, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("c6gd_2xlarge", 8.0, 8.0, 16.0, 15.1, 475.0, 10.0, fast, local, arm64), + new VespaFlavor("c6g_4xlarge", 16.0, 16.0, 32.0, 30.8, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("c6gd_4xlarge", 16.0, 16.0, 32.0, 30.8, 950.0, 10.0, fast, local, arm64), + new VespaFlavor("c6g_8xlarge", 32.0, 32.0, 64.0, 62.3, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("c6gd_8xlarge", 32.0, 32.0, 64.0, 62.3, 1900.0, 10.0, fast, local, arm64), + new VespaFlavor("c6g_12xlarge", 48.0, 48.0, 96.0, 93.0, 16384.0, 10.0, fast, remote, arm64), new VespaFlavor("c6gd_12xlarge", 48.0, 48.0, 96.0, 93.0, 2850.0, 10.0, fast, local, arm64), + new VespaFlavor("c6g_16xlarge", 64.0, 64.0, 128.0, 125.0, 16384.0, 10.0, fast, remote, arm64), new VespaFlavor("c6gd_16xlarge", 64.0, 64.0, 128.0, 125.0, 3800.0, 10.0, fast, local, arm64), - new VespaFlavor("c6id_16xlarge", 64.0, 64.0, 128.0, 123.0, 3800.0, 10.0, fast, local, x86_64), + new VespaFlavor("m7g_8xlarge", 32.0, 32.0, 128.0, 123.0, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("c7g_large", 2.0, 2.0, 4.0, 3.35, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("c7g_xlarge", 4.0, 4.0, 8.0, 7.2, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("c7g_2xlarge", 8.0, 8.0, 16.0, 15.1, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("c7g_4xlarge", 16.0, 16.0, 32.0, 30.8, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("c7g_8xlarge", 32.0, 32.0, 64.0, 62.3, 16384.0, 10.0, fast, remote, arm64), + new VespaFlavor("c7g_12xlarge", 48.0, 48.0, 96.0, 92.6, 16384.0, 10.0, fast, remote, arm64), new VespaFlavor("c7g_16xlarge", 64.0, 64.0, 128.0, 125.0, 16384.0, 10.0, fast, remote, arm64), new VespaFlavor("r5_large", 2.0, 2.0, 16.0, 15.2, 16384.0, 10.0, fast, remote, x86_64), new VespaFlavor("r5d_large", 2.0, 2.0, 16.0, 15.2, 75.0, 10.0, fast, local, x86_64), @@ -89,14 +145,7 @@ public class AwsNodeTypes { new VespaFlavor("r5d_16xlarge", 64.0, 64.0, 512.0, 498.0, 2400.0, 10.0, fast, local, x86_64), new VespaFlavor("r5_24xlarge", 96.0, 96.0, 768.0, 748.0, 16384.0, 10.0, fast, remote, x86_64), new VespaFlavor("r5d_24xlarge", 96.0, 96.0, 768.0, 748.0, 3600.0, 10.0, fast, local, x86_64), - new VespaFlavor("x1_16xlarge", 64.0, 64.0, 976.0, 946.0, 1920.0, 10.0, fast, local, x86_64), - new VespaFlavor("x1_32xlarge", 128.0, 128.0, 1952.0, 1893.0, 3840.0, 10.0, fast, local, x86_64), - new VespaFlavor("x1e_xlarge", 4.0, 4.0, 122.0, 118.0, 120.0, 10.0, fast, local, x86_64), - new VespaFlavor("x1e_2xlarge", 8.0, 8.0, 244.0, 237.0, 240.0, 10.0, fast, local, x86_64), - new VespaFlavor("x1e_4xlarge", 16.0, 16.0, 488.0, 474.0, 480.0, 10.0, fast, local, x86_64), - new VespaFlavor("x1e_8xlarge", 32.0, 32.0, 976.0, 946.0, 960.0, 10.0, fast, local, x86_64), - new VespaFlavor("x1e_16xlarge", 64.0, 64.0, 1952.0, 1893.0, 1920.0, 10.0, fast, local, x86_64), - new VespaFlavor("x1e_32xlarge", 128.0, 128.0, 3904.0, 3786.0, 3840.0, 10.0, fast, local, x86_64), + new VespaFlavor("r6id_xlarge", 4.0, 4.0, 32.0, 30.8, 237.0, 10.0, fast, local, x86_64), new VespaFlavor("z1d_6xlarge", 24.0, 24.0, 192.0, 185.0, 900.0, 10.0, fast, local, x86_64), new VespaFlavor("i4i_large", 2.0, 2.0, 16.0, 15.1, 468.0, 10.0, fast, local, x86_64), new VespaFlavor("i4i_xlarge", 4.0, 4.0, 32.0, 30.5, 937.0, 10.0, fast, local, x86_64), @@ -104,7 +153,9 @@ public class AwsNodeTypes { new VespaFlavor("i4i_4xlarge", 16.0, 16.0, 128.0, 123.0, 3750.0, 10.0, fast, local, x86_64), new VespaFlavor("i4i_8xlarge", 32.0, 32.0, 256.0, 246.0, 7500.0, 10.0, fast, local, x86_64), new VespaFlavor("i4i_16xlarge", 64.0, 64.0, 512.0, 498.0, 15000.0, 10.0, fast, local, x86_64), - new VespaFlavor("i4i_32xlarge", 128.0, 128.0, 1024.0, 996.0, 30000.0, 10.0, fast, local, x86_64)); + new VespaFlavor("i4i_32xlarge", 128.0, 128.0, 1024.0, 996.0, 30000.0, 10.0, fast, local, x86_64), + new VespaFlavor("g4dn_xlarge", 4.0, 4.0, 16.0, 15.1, 125.0, 10.0, fast, local, x86_64, new GpuResources(1, 16.0)), + new VespaFlavor("g4dn_2xlarge", 8.0, 8.0, 32.0, 30.5, 225.0, 10.0, fast, local, x86_64, new GpuResources(1, 16.0))); public static List<VespaFlavor> asVespaFlavors() { return sorted(hostFlavors); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/VespaFlavor.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/VespaFlavor.java index c42b61988e9..2e0a92a38af 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/VespaFlavor.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/VespaFlavor.java @@ -23,9 +23,23 @@ public class VespaFlavor { NodeResources.DiskSpeed diskSpeed, NodeResources.StorageType storageType, NodeResources.Architecture architecture) { + this(name, advertisedVcpu, realVcpu, advertisedMemoryGb, realMemoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, NodeResources.GpuResources.zero()); + } + + public VespaFlavor(String name, + double advertisedVcpu, + double realVcpu, + double advertisedMemoryGb, + double realMemoryGb, + double diskGb, + double bandwidthGbps, + NodeResources.DiskSpeed diskSpeed, + NodeResources.StorageType storageType, + NodeResources.Architecture architecture, + NodeResources.GpuResources gpuResources) { this.name = name; - this.realResources = new NodeResources(realVcpu, realMemoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture); - this.advertisedResources = new NodeResources(advertisedVcpu, advertisedMemoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture); + this.realResources = new NodeResources(realVcpu, realMemoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); + this.advertisedResources = new NodeResources(advertisedVcpu, advertisedMemoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); } public String name() { return name; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index 6dc681ae5c8..b5257e23d9e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -40,6 +40,7 @@ public class LoadBalancerSerializerTest { Optional.of(new LoadBalancerInstance( Optional.of(DomainName.of("lb-host")), Optional.empty(), + Optional.empty(), Optional.of(new DnsZone("zone-id-1")), Set.of(4080, 4443), Set.of("10.2.3.4/24"), @@ -73,6 +74,7 @@ public class LoadBalancerSerializerTest { Optional.of(new LoadBalancerInstance( Optional.empty(), Optional.of("1.2.3.4"), + Optional.of("fd00::1"), Optional.of(new DnsZone("zone-id-1")), Set.of(4443), Set.of("10.2.3.4/24", "12.3.2.1/30"), @@ -86,6 +88,8 @@ public class LoadBalancerSerializerTest { var serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer)); assertEquals(loadBalancer.id(), serialized.id()); assertEquals(loadBalancer.instance().get().hostname(), serialized.instance().get().hostname()); + assertEquals(loadBalancer.instance().get().ip4Address(), serialized.instance().get().ip4Address()); + assertEquals(loadBalancer.instance().get().ip6Address(), serialized.instance().get().ip6Address()); assertEquals(loadBalancer.instance().get().dnsZone(), serialized.instance().get().dnsZone()); assertEquals(loadBalancer.instance().get().ports(), serialized.instance().get().ports()); assertEquals(loadBalancer.instance().get().networks(), serialized.instance().get().networks()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index ebac6071a14..1ef89f2d53d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -289,6 +289,11 @@ public class DynamicProvisioningTester { return flavorResources.compatibleWith(resources); } + @Override + public boolean satisfies(Flavor flavor, NodeResources resources) { + return hostResourcesCalculator.advertisedResourcesOf(flavor).satisfies(resources); + } + } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json index 928e91861a2..54a0e7e9757 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json @@ -119,6 +119,10 @@ ], "additionalIpAddresses": [], "cloudAccount": "aws:111222333444", - "wireguardPubkey":"lololololololololololololololololololololoo=", + "wireguard": { + "key": "lololololololololololololololololololololoo=", + "timestamp": 456 + }, + "wireguardPubkey": "lololololololololololololololololololololoo=", "wireguardKeyTimestamp": 456 } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json index 72b5483d849..d3f1a8082ae 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json @@ -117,6 +117,10 @@ "ipAddresses": ["127.0.101.1", "::101:1"], "additionalIpAddresses": ["::101:2", "::101:3", "::101:4"], "cloudAccount": "aws:777888999000", + "wireguard": { + "key": "000011112222333344445555666677778888999900c=", + "timestamp": 123 + }, "wireguardPubkey": "000011112222333344445555666677778888999900c=", "wireguardKeyTimestamp": 123 } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json index d0d6df71fc1..404cf9a9a80 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json @@ -118,6 +118,10 @@ "ipAddresses": ["127.0.4.1", "::4:1"], "additionalIpAddresses": [], "cloudAccount": "aws:111222333444", + "wireguard": { + "key": "lololololololololololololololololololololoo=", + "timestamp": 123 + }, "wireguardPubkey": "lololololololololololololololololololololoo=", "wireguardKeyTimestamp": 123 } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json index 7bee06adc87..8e9af7f680f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json @@ -4,7 +4,11 @@ "hostname": "cfg1.yahoo.com", "wireguardPubkey": "lololololololololololololololololololololoo=", "wireguardKeyTimestamp":456, - "ipAddresses": ["::201:1"] + "ipAddresses": ["::201:1"], + "wireguard": { + "key": "lololololololololololololololololololololoo=", + "timestamp": 456 + } } ] } diff --git a/parent/pom.xml b/parent/pom.xml index 50222402b68..d23d12c72c9 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -317,7 +317,7 @@ --> <groupId>org.openrewrite.maven</groupId> <artifactId>rewrite-maven-plugin</artifactId> - <version>5.5.2</version> + <version>5.7.1</version> <configuration> <activeRecipes> <recipe>org.openrewrite.java.testing.junit5.JUnit5BestPractices</recipe> diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 575a52d01fb..d098bdde8b6 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -711,7 +711,7 @@ void Test::requireThatQueryGluesEverythingTogether() { EXPECT_EQUAL(1u, md->getNumTermFields()); query.optimize(); - query.fetchPostings(); + query.fetchPostings(requestContext.getDoom()); SearchIterator::UP search = query.createSearch(*md); ASSERT_TRUE(search.get()); } @@ -744,7 +744,7 @@ void checkQueryAddsLocation(const string &loc_in, const string &loc_out) { MatchData::UP md = mdl.createMatchData(); EXPECT_EQUAL(2u, md->getNumTermFields()); - query.fetchPostings(); + query.fetchPostings(requestContext.getDoom()); SearchIterator::UP search = query.createSearch(*md); ASSERT_TRUE(search.get()); if (!EXPECT_NOT_EQUAL(string::npos, search->asString().find(loc_out))) { @@ -966,7 +966,7 @@ Test::requireThatWhiteListBlueprintCanBeUsed() MatchData::UP md = mdl.createMatchData(); query.optimize(); - query.fetchPostings(); + query.fetchPostings(requestContext.getDoom()); SearchIterator::UP search = query.createSearch(*md); SimpleResult exp = SimpleResult().addHit(1).addHit(5).addHit(7).addHit(11); SimpleResult act; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index be1c8941f65..e1820ece0e3 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -52,22 +52,11 @@ DocsumContext::initState() _docsumState._args.initFromDocsumRequest(req); _docsumState._docsumbuf.clear(); _docsumState._docsumbuf.reserve(req.hits.size()); - for (uint32_t i = 0; i < req.hits.size(); i++) { - _docsumState._docsumbuf.push_back(req.hits[i].docid); + for (const auto & hit : req.hits) { + _docsumState._docsumbuf.push_back(hit.docid); } } -namespace { - -vespalib::Slime::Params -makeSlimeParams(size_t chunkSize) { - Slime::Params params; - params.setChunkSize(chunkSize); - return params; -} - -} - vespalib::Slime::UP DocsumContext::createSlimeReply() { @@ -75,11 +64,11 @@ DocsumContext::createSlimeReply() _docsumState._args.get_fields()); _docsumWriter.initState(_attrMgr, _docsumState, rci); const size_t estimatedChunkSize(std::min(0x200000ul, _docsumState._docsumbuf.size()*0x400ul)); - vespalib::Slime::UP response(std::make_unique<vespalib::Slime>(makeSlimeParams(estimatedChunkSize))); + auto response = std::make_unique<vespalib::Slime>(Slime::Params(estimatedChunkSize)); Cursor & root = response->setObject(); Cursor & array = root.setArray(DOCSUMS); const Symbol docsumSym = response->insert(DOCSUM); - _docsumState._omit_summary_features = (rci.res_class != nullptr) ? rci.res_class->omit_summary_features() : true; + _docsumState._omit_summary_features = (rci.res_class == nullptr) || rci.res_class->omit_summary_features(); uint32_t num_ok(0); for (uint32_t docId : _docsumState._docsumbuf) { if (_request.expired() ) { break; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp index 1528b327747..b8027bff04a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/queryeval/searchable.h> #include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/searchlib/queryeval/irequestcontext.h> #include <vespa/searchlib/query/tree/range.h> #include <vespa/searchlib/query/tree/simplequery.h> @@ -98,7 +99,7 @@ AttributeLimiter::create_search(size_t want_hits, size_t max_group_size, bool st FieldSpecList field; // single field API is protected field.add(FieldSpec(_attribute_name, my_field_id, my_handle)); _blueprint = _searchable_attributes.createBlueprint(_requestContext, field, node); - _blueprint->fetchPostings(ExecuteInfo::create(strictSearch)); + _blueprint->fetchPostings(ExecuteInfo::create(strictSearch, &_requestContext.getDoom())); _estimatedHits.store(_blueprint->getState().estimate().estHits, std::memory_order_relaxed); _blueprint->freeze(); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index b57346611f1..14a238330ba 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -222,6 +222,8 @@ MatchThread::match_loop(MatchTools &tools, HitCollector &hits) !docid_range.empty(); docid_range = scheduler.next_range(thread_id)) { + // Due to some schedulers communicating across threads, it is vital that all complete this + // loop. Do not break out. if (!softDoomed) { uint32_t lastCovered = inner_match_loop<Strategy, do_rank, do_limit, do_share_work, use_rank_drop_limit>(context, tools, docid_range); softDoomed = (lastCovered < docid_range.end); @@ -311,6 +313,41 @@ MatchThread::match_loop_helper(MatchTools &tools, HitCollector &hits) } } +void +MatchThread::secondPhase(MatchTools & tools, HitCollector & hits) { + trace->addEvent(4, "Start second phase rerank"); + auto sorted_hit_seq = matchToolsFactory.should_diversify() + ? hits.getSortedHitSequence(matchParams.arraySize) + : hits.getSortedHitSequence(matchParams.heapSize); + trace->addEvent(5, "Synchronize before second phase rerank"); + WaitTimer get_second_phase_work_timer(wait_time_s); + /** + * All, or none of the threads in the bundle should call communicator.get_second_phase_work and + * communicator.complete_second_phase. + * Avoid early return and handle doom with care. + */ + auto my_work = communicator.get_second_phase_work(sorted_hit_seq, thread_id); + get_second_phase_work_timer.done(); + if (tools.getDoom().hard_doom()) { + my_work.clear(); + } + if (!my_work.empty()) { + tools.setup_second_phase(second_phase_profiler.get()); + DocumentScorer scorer(tools.rank_program(), tools.search()); + scorer.score(my_work); + } + thread_stats.docsReRanked(my_work.size()); + trace->addEvent(5, "Synchronize before rank scaling"); + WaitTimer complete_second_phase_timer(wait_time_s); + auto [kept_hits, ranges] = communicator.complete_second_phase(my_work, thread_id); + complete_second_phase_timer.done(); + hits.setReRankedHits(std::move(kept_hits)); + hits.setRanges(ranges); + if (auto onReRankTask = matchToolsFactory.createOnSecondPhaseTask()) { + onReRankTask->run(hits.getReRankedHits()); + } +} + search::ResultSet::UP MatchThread::findMatches(MatchTools &tools) { @@ -332,34 +369,15 @@ MatchThread::findMatches(MatchTools &tools) } HitCollector hits(matchParams.numDocs, matchParams.arraySize); trace->addEvent(4, "Start match and first phase rank"); + /** + * All, or none of the threads in the bundle must execute the match loop. + * The same goes for secondPhase. + * This is due to all the threads in the bundle needs to meet up and exchange information. + * If not you will have deadlock. + */ match_loop_helper(tools, hits); if (tools.has_second_phase_rank()) { - trace->addEvent(4, "Start second phase rerank"); - auto sorted_hit_seq = matchToolsFactory.should_diversify() - ? hits.getSortedHitSequence(matchParams.arraySize) - : hits.getSortedHitSequence(matchParams.heapSize); - trace->addEvent(5, "Synchronize before second phase rerank"); - WaitTimer get_second_phase_work_timer(wait_time_s); - auto my_work = communicator.get_second_phase_work(sorted_hit_seq, thread_id); - get_second_phase_work_timer.done(); - if (tools.getDoom().hard_doom()) { - my_work.clear(); - } - if (!my_work.empty()) { - tools.setup_second_phase(second_phase_profiler.get()); - DocumentScorer scorer(tools.rank_program(), tools.search()); - scorer.score(my_work); - } - thread_stats.docsReRanked(my_work.size()); - trace->addEvent(5, "Synchronize before rank scaling"); - WaitTimer complete_second_phase_timer(wait_time_s); - auto [kept_hits, ranges] = communicator.complete_second_phase(my_work, thread_id); - complete_second_phase_timer.done(); - hits.setReRankedHits(std::move(kept_hits)); - hits.setRanges(ranges); - if (auto onReRankTask = matchToolsFactory.createOnSecondPhaseTask()) { - onReRankTask->run(hits.getReRankedHits()); - } + secondPhase(tools, hits); } trace->addEvent(4, "Create result set"); return hits.getResultSet(fallback_rank_value()); @@ -463,6 +481,11 @@ MatchThread::run() auto capture_issues = vespalib::Issue::listen(my_issues); trace->addEvent(4, "Start MatchThread::run"); MatchTools::UP matchTools = matchToolsFactory.createMatchTools(); + /** + * All, or none of the threads in the bundle must call findMatches. + * All, or none of the threads in the bundle must call mergeDirector.dualMerge. + * Avoid early return and handle doom with care. + */ search::ResultSet::UP result = findMatches(*matchTools); match_time_s = vespalib::to_s(match_time.elapsed()); resultContext = resultProcessor.createThreadContext(matchTools->getDoom(), thread_id, _distributionKey); @@ -475,6 +498,7 @@ MatchThread::run() result->getNumHits(), resultContext->sort->hasSortData(), bool(resultContext->grouping))); + (void) processToken; // Avoid unused warning get_token_timer.done(); trace->addEvent(5, "Start result processing"); processResult(matchTools->getDoom(), std::move(result), *resultContext); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h index 03ba34eca1f..ad864a98227 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h @@ -113,6 +113,7 @@ private: void match_loop_helper(MatchTools &tools, HitCollector &hits); search::ResultSet::UP findMatches(MatchTools &tools); + void secondPhase(MatchTools & tools, HitCollector & hits); void processResult(const Doom & doom, search::ResultSet::UP result, ResultProcessor::Context &context); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index 5ae671b88cb..758ef35ebc9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -201,9 +201,9 @@ MatchToolsFactory(QueryLimiter & queryLimiter, trace.addEvent(5, "Optimize query execution plan"); _query.optimize(); trace.addEvent(4, "Perform dictionary lookups and posting lists initialization"); - _query.fetchPostings(); + _query.fetchPostings(_requestContext.getDoom()); if (is_search) { - _query.handle_global_filter(searchContext.getDocIdLimit(), + _query.handle_global_filter(_requestContext.getDoom(), searchContext.getDocIdLimit(), _attribute_blueprint_params.global_filter_lower_limit, _attribute_blueprint_params.global_filter_upper_limit, thread_bundle, trace); diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index d0738f1857f..22f6ec9cc88 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -247,13 +247,14 @@ Query::optimize() } void -Query::fetchPostings() +Query::fetchPostings(const vespalib::Doom & doom) { - _blueprint->fetchPostings(search::queryeval::ExecuteInfo::create(true, 1.0)); + _blueprint->fetchPostings(search::queryeval::ExecuteInfo::create(true, &doom)); } void -Query::handle_global_filter(uint32_t docid_limit, double global_filter_lower_limit, double global_filter_upper_limit, +Query::handle_global_filter(const vespalib::Doom & doom, uint32_t docid_limit, + double global_filter_lower_limit, double global_filter_upper_limit, vespalib::ThreadBundle &thread_bundle, search::engine::Trace& trace) { if (!handle_global_filter(*_blueprint, docid_limit, global_filter_lower_limit, global_filter_upper_limit, thread_bundle, &trace)) { @@ -264,7 +265,7 @@ Query::handle_global_filter(uint32_t docid_limit, double global_filter_lower_lim _blueprint = Blueprint::optimize(std::move(_blueprint)); LOG(debug, "blueprint after handle_global_filter:\n%s\n", _blueprint->asString().c_str()); // strictness may change if optimized order changed: - fetchPostings(); + fetchPostings(doom); } bool diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h index b0299307e92..1a3136042a7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.h +++ b/searchcore/src/vespa/searchcore/proton/matching/query.h @@ -98,9 +98,10 @@ public: * test to verify the original query without optimization. **/ void optimize(); - void fetchPostings(); + void fetchPostings(const vespalib::Doom & doom); - void handle_global_filter(uint32_t docid_limit, double global_filter_lower_limit, double global_filter_upper_limit, + void handle_global_filter(const vespalib::Doom & doom, uint32_t docid_limit, + double global_filter_lower_limit, double global_filter_upper_limit, vespalib::ThreadBundle &thread_bundle, search::engine::Trace& trace); /** diff --git a/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp b/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp index c7bd0e917f3..c2a39779061 100644 --- a/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp +++ b/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/fuzzy/levenshtein_dfa.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/text/utf8.h> #include <filesystem> #include <fstream> #include <iostream> @@ -26,13 +27,24 @@ using namespace search::attribute; using namespace search; using vespalib::FuzzyMatcher; using vespalib::datastore::AtomicEntryRef; +using vespalib::datastore::EntryRef; using vespalib::fuzzy::LevenshteinDfa; +using vespalib::Utf8Reader; +using vespalib::Utf8Writer; using StringEnumStore = EnumStoreT<const char*>; using DictionaryEntry = std::pair<std::string, size_t>; using RawDictionary = std::vector<DictionaryEntry>; using StringVector = std::vector<std::string>; +namespace { + +const char* char_from_u8(const char8_t* p) { + return reinterpret_cast<const char*>(p); +} + +} + RawDictionary read_dictionary() { @@ -109,11 +121,11 @@ struct MatchStats { template <bool collect_matches> void -brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, MatchStats& stats, StringVector& matched_words) +brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, bool cased, MatchStats& stats, StringVector& matched_words) { auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); vespalib::Timer timer; - FuzzyMatcher matcher(target, 2, 0, false); + FuzzyMatcher matcher(target, 2, prefix_size, cased); auto itr = view.begin(); size_t matches = 0; size_t seeks = 0; @@ -133,15 +145,33 @@ brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumS template <bool collect_matches> void -dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, MatchStats& stats, StringVector& matched_words) +dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, bool cased, MatchStats& stats, StringVector& matched_words) { auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); vespalib::Timer timer; - DfaFuzzyMatcher matcher(target, 2, false, LevenshteinDfa::DfaType::Explicit); - auto itr = view.begin(); + DfaFuzzyMatcher matcher(target, 2, prefix_size, cased, LevenshteinDfa::DfaType::Explicit); + Utf8Reader reader(vespalib::stringref(target.data(), target.size())); + std::string target_copy; + Utf8Writer<std::string> writer(target_copy); + for (size_t pos = 0; pos < prefix_size && reader.hasMore(); ++pos) { + auto code_point = reader.getChar(); + writer.putChar(code_point); + } + auto prefix_cmp = store.make_folded_comparator_prefix(target_copy.c_str()); + auto itr = prefix_size > 0 ? view.lowerBound(AtomicEntryRef(), prefix_cmp) : view.begin(); + auto itr_end = itr; + if (itr_end.valid()) { + if (prefix_size > 0) { + if (!prefix_cmp.less(EntryRef(), itr_end.getKey().load_relaxed())) { + itr_end.seekPast(AtomicEntryRef(), prefix_cmp); + } + } else { + itr_end.end(); + } + } size_t matches = 0; size_t seeks = 0; - while (itr.valid()) { + while (itr != itr_end) { auto word = store.get_value(itr.getKey().load_relaxed()); if (matcher.is_match(word, itr, store.get_data_store())) { ++itr; @@ -156,10 +186,58 @@ dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& st stats.add_sample(matches, seeks, timer.elapsed()); } -struct DfaFuzzyMatcherTest : public ::testing::Test { +template <bool collect_matches> +void +dfa_fuzzy_match_in_dictionary_no_skip(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, bool cased, MatchStats& stats, StringVector& matched_words) +{ + auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); + vespalib::Timer timer; + DfaFuzzyMatcher matcher(target, 2, prefix_size, cased, LevenshteinDfa::DfaType::Explicit); + auto itr = view.begin(); + size_t matches = 0; + size_t seeks = 0; + for (;itr.valid(); ++itr) { + auto word = store.get_value(itr.getKey().load_relaxed()); + if (matcher.is_match(word)) { + ++matches; + if (collect_matches) { + matched_words.push_back(word); + } + } else { + ++seeks; + } + } + stats.add_sample(matches, seeks, timer.elapsed()); +} + +struct TestParam +{ + vespalib::string _name; + bool _cased; + + TestParam(vespalib::string name, bool cased) + : _name(std::move(name)), + _cased(cased) + { + } + TestParam(const TestParam&); + ~TestParam(); +}; + +TestParam::TestParam(const TestParam&) = default; + +TestParam::~TestParam() = default; + +std::ostream& operator<<(std::ostream& os, const TestParam& param) +{ + os << param._name; + return os; +} + +struct DfaFuzzyMatcherTest : public ::testing::TestWithParam<TestParam> { StringEnumStore store; DfaFuzzyMatcherTest() - : store(true, DictionaryConfig(DictionaryConfig::Type::BTREE, DictionaryConfig::Match::UNCASED)) + : store(true, DictionaryConfig(DictionaryConfig::Type::BTREE, GetParam()._cased ? DictionaryConfig::Match::CASED : DictionaryConfig::Match::UNCASED)) {} void populate_dictionary(const StringVector& words) { auto updater = store.make_batch_updater(); @@ -170,18 +248,31 @@ struct DfaFuzzyMatcherTest : public ::testing::Test { updater.commit(); store.freeze_dictionary(); } - void expect_matches(std::string_view target, const StringVector& exp_matches) { + void expect_prefix_matches(std::string_view target, uint32_t prefix_size, const StringVector& exp_matches) { MatchStats stats; StringVector brute_force_matches; StringVector dfa_matches; - brute_force_fuzzy_match_in_dictionary<true>(target, store, stats, brute_force_matches); - dfa_fuzzy_match_in_dictionary<true>(target, store, stats, dfa_matches); + StringVector dfa_no_skip_matches; + bool cased = GetParam()._cased; + SCOPED_TRACE(target); + brute_force_fuzzy_match_in_dictionary<true>(target, store, prefix_size, cased, stats, brute_force_matches); + dfa_fuzzy_match_in_dictionary<true>(target, store, prefix_size, cased, stats, dfa_matches); + dfa_fuzzy_match_in_dictionary_no_skip<true>(target, store, prefix_size, cased, stats, dfa_no_skip_matches); EXPECT_EQ(exp_matches, brute_force_matches); EXPECT_EQ(exp_matches, dfa_matches); + EXPECT_EQ(exp_matches, dfa_no_skip_matches); + } + void expect_matches(std::string_view target, const StringVector& exp_matches) { + expect_prefix_matches(target, 0, exp_matches); } }; -TEST_F(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary) +INSTANTIATE_TEST_SUITE_P(DfaFuzzyMatcherMultiTest, + DfaFuzzyMatcherTest, + testing::Values(TestParam("uncased", false), TestParam("cased", true)), + testing::PrintToStringParamName()); + +TEST_P(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary) { StringVector words = { "board", "boat", "bob", "door", "food", "foot", "football", "foothill", "for", "forbid", "force", "ford", "forearm", "forecast", "forest" }; @@ -194,23 +285,67 @@ TEST_F(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary) expect_matches("forcecast", {"forecast"}); } +TEST_P(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary_with_prefix_size) +{ + bool cased = GetParam()._cased; + StringVector words = { "board", "boat", "bob", "door", "food", "foot", "football", "foothill", + "for", "forbid", "force", "ford", "forearm", "forecast", "forest", "H", "HA", "h", "ha", char_from_u8(u8"Ørn"), char_from_u8(u8"øre"), char_from_u8(u8"Ås"), char_from_u8(u8"ås")}; + populate_dictionary(words); + expect_prefix_matches("a", 1, {}); + expect_prefix_matches("b", 1, {"bob"}); + expect_prefix_matches("board", 1, {"board", "boat"}); + expect_prefix_matches("c", 1, {}); + expect_prefix_matches("food", 1, {"food", "foot", "for", "ford"}); + expect_prefix_matches("food", 2, {"food", "foot", "for", "ford"}); + expect_prefix_matches("food", 3, {"food", "foot"}); + expect_prefix_matches("foothill", 1, {"football", "foothill"}); + expect_prefix_matches("for", 1, {"food", "foot", "for", "force", "ford"}); + expect_prefix_matches("for", 2, {"food", "foot", "for", "force", "ford"}); + expect_prefix_matches("for", 3, {"for", "force", "ford"}); + expect_prefix_matches("force", 1, {"for", "force", "ford"}); + expect_prefix_matches("forcecast", 1, {"forecast"}); + expect_prefix_matches("forcecast", 4, {}); + expect_prefix_matches("z", 1, {}); + if (cased) { + expect_prefix_matches("h", 1, {"h", "ha"}); + expect_prefix_matches(char_from_u8(u8"Ø"), 1, {char_from_u8(u8"Ørn")}); + expect_prefix_matches(char_from_u8(u8"ø"), 1, {char_from_u8(u8"øre")}); + expect_prefix_matches(char_from_u8(u8"å"), 1, {char_from_u8(u8"ås")}); + /* Corner case: prefix length > target length means exact match */ + expect_prefix_matches("h", 2, {"h"}); + } else { + expect_prefix_matches("h", 1, {"H", "h", "HA", "ha"}); + expect_prefix_matches(char_from_u8(u8"ø"), 1, {char_from_u8(u8"øre"), char_from_u8(u8"Ørn")}); + expect_prefix_matches(char_from_u8(u8"å"), 1, {char_from_u8(u8"Ås"), char_from_u8(u8"ås")}); + /* Corner case: prefix length > target length means exact match */ + expect_prefix_matches("h", 2, {"H", "h"}); + } +} + void -benchmark_fuzzy_match_in_dictionary(const StringEnumStore& store, const RawDictionary& dict, size_t words_to_match, bool dfa_algorithm) +benchmark_fuzzy_match_in_dictionary(const StringEnumStore& store, const RawDictionary& dict, size_t words_to_match, bool cased, bool dfa_algorithm) { MatchStats stats; StringVector dummy; for (size_t i = 0; i < std::min(words_to_match, dict.size()); ++i) { const auto& entry = dict[i]; if (dfa_algorithm) { - dfa_fuzzy_match_in_dictionary<false>(entry.first, store, stats, dummy); + dfa_fuzzy_match_in_dictionary<false>(entry.first, store, 0, cased, stats, dummy); } else { - brute_force_fuzzy_match_in_dictionary<false>(entry.first, store, stats, dummy); + brute_force_fuzzy_match_in_dictionary<false>(entry.first, store, 0, cased, stats, dummy); } } std::cout << (dfa_algorithm ? "DFA:" : "Brute force:") << " samples=" << stats.samples << ", avg_matches=" << stats.avg_matches() << ", avg_seeks=" << stats.avg_seeks() << ", avg_elapsed_ms=" << stats.avg_elapsed_ms() << std::endl; } -TEST_F(DfaFuzzyMatcherTest, benchmark_fuzzy_match_in_dictionary) +using DfaFuzzyMatcherBenchmarkTest = DfaFuzzyMatcherTest; + +INSTANTIATE_TEST_SUITE_P(DfaFuzzyMatcherBenchmarkMultiTest, + DfaFuzzyMatcherBenchmarkTest, + testing::Values(TestParam("uncased", false)), + testing::PrintToStringParamName()); + +TEST_P(DfaFuzzyMatcherBenchmarkTest, benchmark_fuzzy_match_in_dictionary) { if (!benchmarking_enabled()) { GTEST_SKIP() << "benchmarking not enabled"; @@ -219,8 +354,9 @@ TEST_F(DfaFuzzyMatcherTest, benchmark_fuzzy_match_in_dictionary) populate_dictionary(to_string_vector(dict)); std::cout << "Unique words: " << store.get_num_uniques() << std::endl; sort_by_freq(dict); - benchmark_fuzzy_match_in_dictionary(store, dict, dfa_words_to_match, true); - benchmark_fuzzy_match_in_dictionary(store, dict, brute_force_words_to_match, false); + bool cased = GetParam()._cased; + benchmark_fuzzy_match_in_dictionary(store, dict, dfa_words_to_match, cased, true); + benchmark_fuzzy_match_in_dictionary(store, dict, brute_force_words_to_match, cased, false); } int diff --git a/searchlib/src/tests/attribute/document_weight_or_filter_search/document_weight_or_filter_search_test.cpp b/searchlib/src/tests/attribute/document_weight_or_filter_search/document_weight_or_filter_search_test.cpp index b9c70d76934..1fd9dde09c7 100644 --- a/searchlib/src/tests/attribute/document_weight_or_filter_search/document_weight_or_filter_search_test.cpp +++ b/searchlib/src/tests/attribute/document_weight_or_filter_search/document_weight_or_filter_search_test.cpp @@ -24,14 +24,14 @@ class DocumentWeightOrFilterSearchTest : public ::testing::Test { uint32_t _range_end; public: DocumentWeightOrFilterSearchTest(); - ~DocumentWeightOrFilterSearchTest(); + ~DocumentWeightOrFilterSearchTest() override; void inc_generation(); size_t num_trees() const { return _trees.size(); } Iterator get_tree(size_t idx) const { if (idx < _trees.size()) { return _postings.beginFrozen(_trees[idx]); } else { - return Iterator(); + return {}; } } void ensure_tree(size_t idx) { @@ -39,13 +39,13 @@ public: _trees.resize(idx + 1); } } - void add_tree(size_t idx, std::vector<uint32_t> keys) { + void add_tree(size_t idx, const std::vector<uint32_t>& keys) { ensure_tree(idx); std::vector<KeyData> adds; std::vector<uint32_t> removes; adds.reserve(keys.size()); for (auto& key : keys) { - adds.emplace_back(KeyData(key, 1)); + adds.emplace_back(key, 1); } _postings.apply(_trees[idx], adds.data(), adds.data() + adds.size(), removes.data(), removes.data() + removes.size()); } @@ -67,7 +67,7 @@ public: return result; }; - std::vector<uint32_t> eval_daat(SearchIterator &iterator) { + std::vector<uint32_t> eval_daat(SearchIterator &iterator) const { std::vector<uint32_t> result; uint32_t doc_id = _range_start; while (doc_id < _range_end) { @@ -81,7 +81,7 @@ public: return result; } - std::vector<uint32_t> frombv(const BitVector &bv) { + std::vector<uint32_t> frombv(const BitVector &bv) const { std::vector<uint32_t> result; uint32_t doc_id = _range_start; doc_id = bv.getNextTrueBit(doc_id); @@ -93,7 +93,7 @@ public: return result; } - std::unique_ptr<BitVector> tobv(std::vector<uint32_t> values) { + std::unique_ptr<BitVector> tobv(const std::vector<uint32_t> & values) const { auto bv = BitVector::create(_range_start, _range_end); for (auto value : values) { bv->setBit(value); @@ -102,7 +102,7 @@ public: return bv; } - void expect_result(std::vector<uint32_t> exp, std::vector<uint32_t> act) + static void expect_result(const std::vector<uint32_t> & exp, const std::vector<uint32_t> & act) { EXPECT_EQ(exp, act); } @@ -227,7 +227,7 @@ public: } _test.inc_generation(); } - ~Verifier() { + ~Verifier() override { for (uint32_t tree_id = 0; tree_id < _test.num_trees(); ++tree_id) { _test.clear_tree(tree_id); } diff --git a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp index d7a854e0afc..6c6f05fd5e2 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp @@ -29,14 +29,14 @@ using namespace search::attribute::test; namespace { void -setupAttributeManager(MockAttributeManager &manager) +setupAttributeManager(MockAttributeManager &manager, bool isFilter) { AttributeVector::DocId docId; { - AttributeVector::SP attr_sp = AttributeFactory::createAttribute("integer", Config(BasicType("int64"))); + AttributeVector::SP attr_sp = AttributeFactory::createAttribute("integer", Config(BasicType("int64")).setIsFilter(isFilter)); manager.addAttribute(attr_sp); - IntegerAttribute *attr = (IntegerAttribute*)(attr_sp.get()); + auto *attr = (IntegerAttribute*)(attr_sp.get()); for (size_t i = 1; i < 10; ++i) { attr->addDoc(docId); assert(i == docId); @@ -45,10 +45,10 @@ setupAttributeManager(MockAttributeManager &manager) } } { - AttributeVector::SP attr_sp = AttributeFactory::createAttribute("string", Config(BasicType("string"))); + AttributeVector::SP attr_sp = AttributeFactory::createAttribute("string", Config(BasicType("string")).setIsFilter(isFilter)); manager.addAttribute(attr_sp); - StringAttribute *attr = (StringAttribute*)(attr_sp.get()); + auto *attr = (StringAttribute*)(attr_sp.get()); for (size_t i = 1; i < 10; ++i) { attr->addDoc(docId); assert(i == docId); @@ -58,9 +58,9 @@ setupAttributeManager(MockAttributeManager &manager) } { AttributeVector::SP attr_sp = AttributeFactory::createAttribute( - "multi", Config(BasicType("int64"), search::attribute::CollectionType("array"))); + "multi", Config(BasicType("int64"), search::attribute::CollectionType("array")).setIsFilter(isFilter)); manager.addAttribute(attr_sp); - IntegerAttribute *attr = (IntegerAttribute*)(attr_sp.get()); + auto *attr = (IntegerAttribute*)(attr_sp.get()); for (size_t i = 1; i < 10; ++i) { attr->addDoc(docId); assert(i == docId); @@ -78,35 +78,43 @@ struct WS { TermFieldHandle handle; std::vector<std::pair<std::string, uint32_t> > tokens; - WS(IAttributeManager & manager) : attribute_manager(manager), layout(), handle(layout.allocTermField(fieldId)), tokens() { + explicit WS(IAttributeManager & manager) + : attribute_manager(manager), + layout(), handle(layout.allocTermField(fieldId)), + tokens() + { MatchData::UP tmp = layout.createMatchData(); ASSERT_TRUE(tmp->resolveTermField(handle)->getFieldId() == fieldId); } WS &add(const std::string &token, uint32_t weight) { - tokens.push_back(std::make_pair(token, weight)); + tokens.emplace_back(token, weight); return *this; } Node::UP createNode() const { - SimpleWeightedSetTerm *node = new SimpleWeightedSetTerm(tokens.size(), "view", 0, Weight(0)); - for (size_t i = 0; i < tokens.size(); ++i) { - node->addTerm(tokens[i].first, Weight(tokens[i].second)); + auto *node = new SimpleWeightedSetTerm(tokens.size(), "view", 0, Weight(0)); + for (const auto & token : tokens) { + node->addTerm(token.first, Weight(token.second)); } return Node::UP(node); } - bool isGenericSearch(Searchable &searchable, const std::string &field, bool strict) const { + SearchIterator::UP + createSearch(Searchable &searchable, const std::string &field, bool strict) const { AttributeContext ac(attribute_manager); FakeRequestContext requestContext(&ac); MatchData::UP md = layout.createMatchData(); Node::UP node = createNode(); FieldSpecList fields; - fields.add(FieldSpec(field, fieldId, handle)); + fields.add(FieldSpec(field, fieldId, handle, ac.getAttribute(field)->getIsFilter())); queryeval::Blueprint::UP bp = searchable.createBlueprint(requestContext, fields, *node); bp->fetchPostings(queryeval::ExecuteInfo::create(strict)); SearchIterator::UP sb = bp->createSearch(*md, strict); - return (dynamic_cast<WeightedSetTermSearch*>(sb.get()) != 0); + return sb; + } + bool isWeightedSetTermSearch(Searchable &searchable, const std::string &field, bool strict) const { + return dynamic_cast<WeightedSetTermSearch *>(createSearch(searchable, field, strict).get()) != nullptr; } FakeResult search(Searchable &searchable, const std::string &field, bool strict) const { @@ -140,23 +148,58 @@ struct WS { } // namespace <unnamed> +void test_tokens(bool isFilter, const std::vector<uint32_t> & docs) { + MockAttributeManager manager; + setupAttributeManager(manager, isFilter); + AttributeBlueprintFactory adapter; + + FakeResult expect = FakeResult(); + WS ws = WS(manager); + for (uint32_t doc : docs) { + auto docS = vespalib::stringify(doc); + int32_t weight = doc * 10; + expect.doc(doc).weight(weight).pos(0); + ws.add(docS, weight); + } + + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "integer", true)); + EXPECT_TRUE(!ws.isWeightedSetTermSearch(adapter, "integer", false)); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "string", true)); + EXPECT_TRUE(!ws.isWeightedSetTermSearch(adapter, "string", false)); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "multi", true)); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "multi", false)); + + EXPECT_EQUAL(expect, ws.search(adapter, "integer", true)); + EXPECT_EQUAL(expect, ws.search(adapter, "integer", false)); + EXPECT_EQUAL(expect, ws.search(adapter, "string", true)); + EXPECT_EQUAL(expect, ws.search(adapter, "string", false)); + EXPECT_EQUAL(expect, ws.search(adapter, "multi", true)); + EXPECT_EQUAL(expect, ws.search(adapter, "multi", false)); +} TEST("attribute_weighted_set_test") { + test_tokens(false, {3, 5, 7}); + test_tokens(true, {3, 5, 7}); + test_tokens(false, {3}); +} + +TEST("attribute_weighted_set_single_token_filter_lifted_out") { MockAttributeManager manager; - setupAttributeManager(manager); + setupAttributeManager(manager, true); AttributeBlueprintFactory adapter; - FakeResult expect = FakeResult() - .doc(3).elem(0).weight(30).pos(0) - .doc(5).elem(0).weight(50).pos(0) - .doc(7).elem(0).weight(70).pos(0); - WS ws = WS(manager).add("7", 70).add("5", 50).add("3", 30); - - EXPECT_TRUE(ws.isGenericSearch(adapter, "integer", true)); - EXPECT_TRUE(!ws.isGenericSearch(adapter, "integer", false)); - EXPECT_TRUE(ws.isGenericSearch(adapter, "string", true)); - EXPECT_TRUE(!ws.isGenericSearch(adapter, "string", false)); - EXPECT_TRUE(ws.isGenericSearch(adapter, "multi", true)); - EXPECT_TRUE(ws.isGenericSearch(adapter, "multi", false)); + FakeResult expect = FakeResult().doc(3).elem(0).weight(30).pos(0); + WS ws = WS(manager).add("3", 30); + + EXPECT_EQUAL("search::FilterAttributeIteratorStrict<search::attribute::SingleNumericSearchContext<long, search::attribute::NumericMatcher<long> > >", + ws.createSearch(adapter, "integer", true)->getClassName()); + EXPECT_EQUAL("search::FilterAttributeIteratorT<search::attribute::SingleNumericSearchContext<long, search::attribute::NumericMatcher<long> > >", + ws.createSearch(adapter, "integer", false)->getClassName()); + EXPECT_EQUAL("search::FilterAttributeIteratorStrict<search::attribute::SingleEnumSearchContext<char const*, search::attribute::StringSearchContext> >", + ws.createSearch(adapter, "string", true)->getClassName()); + EXPECT_EQUAL("search::FilterAttributeIteratorT<search::attribute::SingleEnumSearchContext<char const*, search::attribute::StringSearchContext> >", + ws.createSearch(adapter, "string", false)->getClassName()); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "multi", true)); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "multi", false)); EXPECT_EQUAL(expect, ws.search(adapter, "integer", true)); EXPECT_EQUAL(expect, ws.search(adapter, "integer", false)); diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp index 40d4b20aaf2..ac1042dda6c 100644 --- a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp +++ b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributeiterators.h> #include <vespa/searchlib/attribute/flagattribute.h> +#include <vespa/searchlib/attribute/postinglistsearchcontext.h> #include <vespa/searchlib/attribute/searchcontextelementiterator.h> #include <vespa/searchlib/attribute/singleboolattribute.h> #include <vespa/searchlib/attribute/stringbase.h> @@ -1424,6 +1425,25 @@ SearchContextTest::testPrefixSearch(const vespalib::string& name, const Config& } } } + + // Long range of prefixes with unique strings that causes + // PostingListFoldedSearchContextT<DataT>::countHits() to populate + // partial vector of posting indexes, with scan resumed by + // fillArray or fillBitVector. + auto& vec = dynamic_cast<StringAttribute &>(*attr.get()); + uint32_t old_size = attr->getNumDocs(); + constexpr uint32_t longrange_values = search::attribute::PostingListFoldedSearchContextT<int32_t>::MAX_POSTING_INDEXES_SIZE + 100; + attr->addDocs(longrange_values); + DocSet exp_longrange; + for (uint32_t i = 0; i < longrange_values; ++i) { + vespalib::asciistream ss; + ss << "lpref" << i; + vespalib::string sss(ss.str()); + exp_longrange.put(old_size + i); + vec.update(old_size + i, vespalib::string(ss.str()).c_str()); + } + attr->commit(); + performSearch(*attr, "lpref", exp_longrange, TermType::PREFIXTERM); } diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index e9d0f8cb736..52329f31ba7 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -389,8 +389,8 @@ testSingleValue(Attribute & svsa, Config &cfg) TEST("testSingleValue") { EXPECT_EQUAL(24u, sizeof(SearchContext)); - EXPECT_EQUAL(40u, sizeof(StringSearchHelper)); - EXPECT_EQUAL(96u, sizeof(attribute::SingleStringEnumSearchContext)); + EXPECT_EQUAL(48u, sizeof(StringSearchHelper)); + EXPECT_EQUAL(104u, sizeof(attribute::SingleStringEnumSearchContext)); { Config cfg(BasicType::STRING, CollectionType::SINGLE); SingleValueStringAttribute svsa("svsa", cfg); diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index ef0fd56840a..c617db871a7 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -128,9 +128,9 @@ TEST("test And propagates updated histestimate") { const RememberExecuteInfo & child = dynamic_cast<const RememberExecuteInfo &>(bp.getChild(i)); EXPECT_EQUAL((i == 0), child.executeInfo.isStrict()); } - EXPECT_EQUAL(1.0, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(0)).executeInfo.hitRate()); - EXPECT_EQUAL(1.0/250, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(1)).executeInfo.hitRate()); - EXPECT_EQUAL(1.0/(250*25), dynamic_cast<const RememberExecuteInfo &>(bp.getChild(2)).executeInfo.hitRate()); + EXPECT_EQUAL(1.0f, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(0)).executeInfo.hitRate()); + EXPECT_EQUAL(1.0f/250, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(1)).executeInfo.hitRate()); + EXPECT_EQUAL(1.0f/(250*25), dynamic_cast<const RememberExecuteInfo &>(bp.getChild(2)).executeInfo.hitRate()); } TEST("test And Blueprint") { diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 1519bb14554..71ea2a67299 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -337,10 +337,7 @@ public: if (tfmda.size() == 1) { // search in exactly one field fef::TermFieldMatchData &tfmd = *tfmda[0]; - return search::common::create_location_iterator(tfmd, - _attribute.getNumDocs(), - strict, - _location); + return common::create_location_iterator(tfmd, _attribute.getNumDocs(), strict, _location); } else { LOG(debug, "wrong size tfmda: %zu (fallback to old location iterator)\n", tfmda.size()); } @@ -485,6 +482,9 @@ DirectWeightedSetBlueprint<SearchType>::createLeafSearch(const TermFieldMatchDat _attr.create(r.posting_idx, iterators); } bool field_is_filter = getState().fields()[0].isFilter(); + if (field_is_filter && tfmda[0]->isNotNeeded()) { + return attribute::DocumentWeightOrFilterSearch::create(std::move(iterators)); + } return SearchType::create(*tfmda[0], field_is_filter, _weights, std::move(iterators)); } diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp index 108128eeb39..ea9dc9b1948 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp @@ -30,7 +30,7 @@ protected: const attribute::IAttributeVector &attribute() const { return _attr; } public: - UseAttr(const attribute::IAttributeVector & attr) + explicit UseAttr(const attribute::IAttributeVector & attr) : _attr(attr) {} }; @@ -40,7 +40,7 @@ class UseStringEnum : public UseAttr { public: using TokenT = uint32_t; - UseStringEnum(const IAttributeVector & attr) + explicit UseStringEnum(const IAttributeVector & attr) : UseAttr(attr) {} auto mapToken(const ISearchContext &context) const { return attribute().findFoldedEnums(context.queryTerm()->getTerm()); @@ -56,7 +56,7 @@ class UseInteger : public UseAttr { public: using TokenT = uint64_t; - UseInteger(const IAttributeVector & attr) : UseAttr(attr) {} + explicit UseInteger(const IAttributeVector & attr) : UseAttr(attr) {} std::vector<int64_t> mapToken(const ISearchContext &context) const { std::vector<int64_t> result; Int64Range range(context.getAsIntegerTerm()); @@ -157,6 +157,10 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr assert(tfmda.size() == 1); assert(getState().numFields() == 1); fef::TermFieldMatchData &tfmd = *tfmda[0]; + bool field_is_filter = getState().fields()[0].isFilter(); + if ((tfmd.isNotNeeded() || field_is_filter) && (_contexts.size() == 1)) { + return _contexts[0]->createIterator(&tfmd, strict); + } if (strict) { // use generic weighted set search fef::MatchDataLayout layout; auto handle = layout.allocTermField(tfmd.getFieldId()); @@ -167,7 +171,6 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr // TODO: pass ownership with unique_ptr children[i] = _contexts[i]->createIterator(child_tfmd, true).release(); } - bool field_is_filter = getState().fields()[0].isFilter(); return queryeval::WeightedSetTermSearch::create(children, tfmd, field_is_filter, _weights, std::move(match_data)); } else { // use attribute filter optimization bool isString = (_attr.isStringType() && _attr.hasEnum()); @@ -182,18 +185,16 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr } queryeval::SearchIterator::UP -AttributeWeightedSetBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +AttributeWeightedSetBlueprint::createFilterSearch(bool strict, FilterConstraint) const { - (void) constraint; std::vector<std::unique_ptr<queryeval::SearchIterator>> children; children.reserve(_contexts.size()); for (auto& context : _contexts) { - auto wrapper = std::make_unique<search::queryeval::FilterWrapper>(1); + auto wrapper = std::make_unique<queryeval::FilterWrapper>(1); wrapper->wrap(context->createIterator(wrapper->tfmda()[0], strict)); children.emplace_back(std::move(wrapper)); } - search::queryeval::UnpackInfo unpack_info; - return search::queryeval::OrSearch::create(std::move(children), strict, unpack_info); + return queryeval::OrSearch::create(std::move(children), strict, queryeval::UnpackInfo()); } void diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp index 580c34bd5d0..18f480eebcd 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp +++ b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp @@ -1,17 +1,97 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dfa_fuzzy_matcher.h" +#include <vespa/vespalib/text/utf8.h> +#include <vespa/vespalib/text/lowercase.h> using vespalib::fuzzy::LevenshteinDfa; +using vespalib::LowerCase; +using vespalib::Utf8Reader; +using vespalib::Utf8ReaderForZTS; namespace search::attribute { -DfaFuzzyMatcher::DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, bool cased, LevenshteinDfa::DfaType dfa_type) - : _dfa(vespalib::fuzzy::LevenshteinDfa::build(target, max_edits, (cased ? LevenshteinDfa::Casing::Cased : LevenshteinDfa::Casing::Uncased), dfa_type)), - _successor() +namespace { + +std::vector<uint32_t> +extract_prefix(std::string_view target, uint32_t prefix_size, bool cased) +{ + std::vector<uint32_t> result; + result.reserve(prefix_size); + Utf8Reader reader(vespalib::stringref(target.data(), target.size())); + for (size_t pos = 0; pos < prefix_size && reader.hasMore(); ++pos) { + uint32_t code_point = reader.getChar(); + if (!cased) { + code_point = LowerCase::convert(code_point); + } + result.emplace_back(code_point); + } + return result; +} + +std::string_view +extract_suffix(std::string_view target, uint32_t prefix_size) { + Utf8Reader reader(vespalib::stringref(target.data(), target.size())); + for (size_t pos = 0; pos < prefix_size && reader.hasMore(); ++pos) { + (void) reader.getChar(); + } + std::string_view result = target; + result.remove_prefix(reader.getPos()); + return result; +} + +} + +DfaFuzzyMatcher::DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased, LevenshteinDfa::DfaType dfa_type) + : _dfa(vespalib::fuzzy::LevenshteinDfa::build(extract_suffix(target, prefix_size), max_edits, (cased ? LevenshteinDfa::Casing::Cased : LevenshteinDfa::Casing::Uncased), dfa_type)), + _successor(), + _prefix(extract_prefix(target, prefix_size, cased)), + _prefix_size(prefix_size), + _cased(cased) +{ + _successor = _prefix; } DfaFuzzyMatcher::~DfaFuzzyMatcher() = default; +const char* +DfaFuzzyMatcher::skip_prefix(const char* word) const +{ + Utf8ReaderForZTS reader(word); + size_t pos = 0; + for (; pos < _prefix.size() && reader.hasMore(); ++pos) { + (void) reader.getChar(); + } + assert(pos == _prefix.size()); + return reader.get_current_ptr(); +} + +bool +DfaFuzzyMatcher::is_match(const char* word) const +{ + if (_prefix_size > 0) { + Utf8ReaderForZTS reader(word); + size_t pos = 0; + for (; pos < _prefix.size() && reader.hasMore(); ++pos) { + uint32_t code_point = reader.getChar(); + if (!_cased) { + code_point = LowerCase::convert(code_point); + } + if (code_point != _prefix[pos]) { + break; + } + } + if (!reader.hasMore() && pos == _prefix.size() && pos < _prefix_size) { + return true; + } + if (pos != _prefix_size) { + return false; + } + word = reader.get_current_ptr(); + } + auto match = _dfa.match(word); + return match.matches(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h index fcba13f85a4..8e5b3ce0ccd 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h +++ b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h @@ -5,6 +5,7 @@ #include "dfa_string_comparator.h" #include <vespa/vespalib/datastore/atomic_entry_ref.h> #include <vespa/vespalib/fuzzy/levenshtein_dfa.h> +#include <iostream> namespace search::attribute { @@ -17,22 +18,53 @@ namespace search::attribute { class DfaFuzzyMatcher { private: vespalib::fuzzy::LevenshteinDfa _dfa; - std::vector<uint32_t> _successor; + std::vector<uint32_t> _successor; + std::vector<uint32_t> _prefix; + uint32_t _prefix_size; + bool _cased; + const char* skip_prefix(const char* word) const; public: - DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, bool cased, vespalib::fuzzy::LevenshteinDfa::DfaType dfa_type); + DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased, vespalib::fuzzy::LevenshteinDfa::DfaType dfa_type); ~DfaFuzzyMatcher(); + bool is_match(const char *word) const; + + /* + * If prefix size is nonzero then this variant of is_match() + * should only be called with words that starts with the extracted + * prefix of the target word. + * + * Caller must position iterator at right location using lower bound + * functionality in the dictionary. + */ template <typename DictionaryConstIteratorType> bool is_match(const char* word, DictionaryConstIteratorType& itr, const DfaStringComparator::DataStoreType& data_store) { - auto match = _dfa.match(word, _successor); - if (match.matches()) { - return true; + if (_prefix_size > 0) { + word = skip_prefix(word); + if (_prefix.size() < _prefix_size) { + if (*word == '\0') { + return true; + } + _successor.resize(_prefix.size()); + _successor.emplace_back(1); + } else { + _successor.resize(_prefix.size()); + auto match = _dfa.match(word, _successor); + if (match.matches()) { + return true; + } + } } else { - DfaStringComparator cmp(data_store, _successor); - itr.seek(vespalib::datastore::AtomicEntryRef(), cmp); - return false; + _successor.clear(); + auto match = _dfa.match(word, _successor); + if (match.matches()) { + return true; + } } + DfaStringComparator cmp(data_store, _successor); + itr.seek(vespalib::datastore::AtomicEntryRef(), cmp); + return false; } }; diff --git a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp index 3c0bae00047..97e725649d3 100644 --- a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp +++ b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp @@ -2,21 +2,27 @@ #include "document_weight_or_filter_search.h" #include "iterator_pack.h" +#include <vespa/searchlib/fef/matchdata.h> +#include <vespa/searchlib/queryeval/iterator_pack.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/queryeval/emptysearch.h> +using search::queryeval::SearchIteratorPack; + namespace search::attribute { +template<typename IteratorPack> class DocumentWeightOrFilterSearchImpl : public DocumentWeightOrFilterSearch { - AttributeIteratorPack _children; + IteratorPack _children; + void seek_all(uint32_t docId); public: - DocumentWeightOrFilterSearchImpl(AttributeIteratorPack&& children); - ~DocumentWeightOrFilterSearchImpl(); + explicit DocumentWeightOrFilterSearchImpl(IteratorPack&& children); + ~DocumentWeightOrFilterSearchImpl() override; void doSeek(uint32_t docId) override; - void doUnpack(uint32_t) override; + void doUnpack(uint32_t) override { } void initRange(uint32_t begin, uint32_t end) override { SearchIterator::initRange(begin, end); @@ -32,48 +38,75 @@ public: } std::unique_ptr<BitVector> get_hits(uint32_t begin_id) override { + seek_all(getDocId()); return _children.get_hits(begin_id, getEndId()); } Trinary is_strict() const override { return Trinary::True; } }; -DocumentWeightOrFilterSearchImpl::DocumentWeightOrFilterSearchImpl(AttributeIteratorPack&& children) +template<typename IteratorPack> +DocumentWeightOrFilterSearchImpl<IteratorPack>::DocumentWeightOrFilterSearchImpl(IteratorPack&& children) : DocumentWeightOrFilterSearch(), _children(std::move(children)) { } -DocumentWeightOrFilterSearchImpl::~DocumentWeightOrFilterSearchImpl() = default; +template<typename IteratorPack> +DocumentWeightOrFilterSearchImpl<IteratorPack>::~DocumentWeightOrFilterSearchImpl() = default; +template<typename IteratorPack> void -DocumentWeightOrFilterSearchImpl::doSeek(uint32_t docId) -{ - if (_children.get_docid(0) < docId) { - _children.seek(0, docId); - } - uint32_t min_doc_id = _children.get_docid(0); - for (uint16_t i = 1; i < _children.size(); ++i) { - if (_children.get_docid(i) < docId) { +DocumentWeightOrFilterSearchImpl<IteratorPack>::seek_all(uint32_t docId) { + for (uint16_t i = 0; i < _children.size(); ++i) { + uint32_t next = _children.get_docid(i); + if (next < docId) { _children.seek(i, docId); } - min_doc_id = std::min(min_doc_id, _children.get_docid(i)); - } - setDocId(min_doc_id); + } } +template<typename IteratorPack> void -DocumentWeightOrFilterSearchImpl::doUnpack(uint32_t) +DocumentWeightOrFilterSearchImpl<IteratorPack>::doSeek(uint32_t docId) { + uint32_t min_doc_id = endDocId; + for (uint16_t i = 0; i < _children.size(); ++i) { + uint32_t next = _children.get_docid(i); + if (next < docId) { + next = _children.seek(i, docId); + } + if (next == docId) { + setDocId(next); + return; + } + min_doc_id = std::min(min_doc_id, next); + } + setDocId(min_doc_id); } -std::unique_ptr<search::queryeval::SearchIterator> +std::unique_ptr<queryeval::SearchIterator> DocumentWeightOrFilterSearch::create(std::vector<DocumentWeightIterator>&& children) { if (children.empty()) { return std::make_unique<queryeval::EmptySearch>(); } else { - return std::make_unique<DocumentWeightOrFilterSearchImpl>(AttributeIteratorPack(std::move(children))); + std::sort(children.begin(), children.end(), + [](const auto & a, const auto & b) { return a.size() > b.size(); }); + using OrFilter = DocumentWeightOrFilterSearchImpl<AttributeIteratorPack>; + return std::make_unique<OrFilter>(AttributeIteratorPack(std::move(children))); + } +} + +std::unique_ptr<queryeval::SearchIterator> +DocumentWeightOrFilterSearch::create(const std::vector<SearchIterator *>& children, + std::unique_ptr<fef::MatchData> md) +{ + if (children.empty()) { + return std::make_unique<queryeval::EmptySearch>(); + } else { + using OrFilter = DocumentWeightOrFilterSearchImpl<SearchIteratorPack>; + return std::make_unique<OrFilter>(SearchIteratorPack(children, std::move(md))); } } diff --git a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h index 62be883ab52..19fa20e2d51 100644 --- a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h +++ b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h @@ -3,21 +3,21 @@ #include "i_document_weight_attribute.h" #include <vespa/searchlib/queryeval/searchiterator.h> +namespace search::fef { class MatchData; } namespace search::attribute { /** * Filter iterator on top of document weight iterators with OR semantics used during * calculation of global filter for weighted set terms, wand terms and dot product terms. */ -class DocumentWeightOrFilterSearch : public search::queryeval::SearchIterator +class DocumentWeightOrFilterSearch : public queryeval::SearchIterator { protected: - DocumentWeightOrFilterSearch() - : search::queryeval::SearchIterator() - { - } + DocumentWeightOrFilterSearch() = default; public: - static std::unique_ptr<search::queryeval::SearchIterator> create(std::vector<DocumentWeightIterator>&& children); + static std::unique_ptr<SearchIterator> create(std::vector<DocumentWeightIterator>&& children); + static std::unique_ptr<SearchIterator> create(const std::vector<SearchIterator *>& children, + std::unique_ptr<fef::MatchData> md); }; } diff --git a/searchlib/src/vespa/searchlib/attribute/iterator_pack.cpp b/searchlib/src/vespa/searchlib/attribute/iterator_pack.cpp index 147f56d6d47..3d9e3095536 100644 --- a/searchlib/src/vespa/searchlib/attribute/iterator_pack.cpp +++ b/searchlib/src/vespa/searchlib/attribute/iterator_pack.cpp @@ -5,6 +5,14 @@ namespace search { +AttributeIteratorPack::~AttributeIteratorPack() = default; + +AttributeIteratorPack::AttributeIteratorPack(std::vector<DocumentWeightIterator> &&children) + : _children(std::move(children)) +{ + assert(_children.size() < 0x10000); +} + std::unique_ptr<BitVector> AttributeIteratorPack::get_hits(uint32_t begin_id, uint32_t end_id) { BitVector::UP result(BitVector::create(begin_id, end_id)); @@ -17,9 +25,9 @@ AttributeIteratorPack::or_hits_into(BitVector &result, uint32_t begin_id) { for (size_t i = 0; i < size(); ++i) { uint32_t docId = get_docid(i); if (begin_id > docId) { - seek(i, begin_id); + docId = seek(i, begin_id); } - for (docId = get_docid(i); docId < result.size(); docId = next(i)) { + for (uint32_t limit = result.size(); docId < limit; docId = next(i)) { result.setBit(docId); } } diff --git a/searchlib/src/vespa/searchlib/attribute/iterator_pack.h b/searchlib/src/vespa/searchlib/attribute/iterator_pack.h index e042aab5eae..80e4c227860 100644 --- a/searchlib/src/vespa/searchlib/attribute/iterator_pack.h +++ b/searchlib/src/vespa/searchlib/attribute/iterator_pack.h @@ -15,12 +15,12 @@ private: std::vector<DocumentWeightIterator> _children; public: - AttributeIteratorPack() : _children() {} + AttributeIteratorPack() noexcept : _children() {} AttributeIteratorPack(AttributeIteratorPack &&rhs) noexcept = default; AttributeIteratorPack &operator=(AttributeIteratorPack &&rhs) noexcept = default; - explicit AttributeIteratorPack(std::vector<DocumentWeightIterator> &&children) - : _children(std::move(children)) {} + explicit AttributeIteratorPack(std::vector<DocumentWeightIterator> &&children); + ~AttributeIteratorPack(); uint32_t get_docid(uint16_t ref) const { return _children[ref].valid() ? _children[ref].getKey() : endDocId; @@ -41,7 +41,7 @@ public: std::unique_ptr<BitVector> get_hits(uint32_t begin_id, uint32_t end_id); void or_hits_into(BitVector &result, uint32_t begin_id); - size_t size() const { return _children.size(); } + uint16_t size() const noexcept { return _children.size(); } void initRange(uint32_t begin, uint32_t end) { (void) end; for (auto &child: _children) { diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp index 12c887eb407..2b1d4fa3286 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp @@ -27,7 +27,8 @@ PostingListSearchContext(const IEnumStoreDictionary& dictionary, bool has_btree_ _FSTC(0.0), _PLSTC(0.0), _hasWeight(hasWeight), - _useBitVector(useBitVector) + _useBitVector(useBitVector), + _counted_hits() { } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h index 05ccedb39ec..8472d3897a0 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h @@ -10,9 +10,11 @@ #include <vespa/searchcommon/attribute/search_context_params.h> #include <vespa/searchcommon/common/range.h> #include <vespa/searchlib/query/query_term_ucs4.h> -#include <vespa/vespalib/util/regexp.h> +#include <vespa/searchlib/queryeval/executeinfo.h> #include <vespa/vespalib/fuzzy/fuzzy_matcher.h> +#include <vespa/vespalib/util/regexp.h> #include <regex> +#include <optional> namespace search::attribute { @@ -30,24 +32,30 @@ protected: using Dictionary = EnumPostingTree; using DictionaryConstIterator = Dictionary::ConstIterator; using FrozenDictionary = Dictionary::FrozenView; + using EntryRef = vespalib::datastore::EntryRef; using EnumIndex = IEnumStore::Index; - const IEnumStoreDictionary & _dictionary; - const ISearchContext &_baseSearchCtx; - const BitVector *_bv; // bitvector if _useBitVector has been set - const FrozenDictionary _frozenDictionary; - DictionaryConstIterator _lowerDictItr; - DictionaryConstIterator _upperDictItr; - uint64_t _numValues; // attr.getStatus().getNumValues(); - uint32_t _uniqueValues; - uint32_t _docIdLimit; - uint32_t _dictSize; - vespalib::datastore::EntryRef _pidx; - vespalib::datastore::EntryRef _frozenRoot; // Posting list in tree form - float _FSTC; // Filtering Search Time Constant - float _PLSTC; // Posting List Search Time Constant - bool _hasWeight; - bool _useBitVector; + static constexpr long MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION = 100; + static constexpr long MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 20; + static constexpr long MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 10; + + const IEnumStoreDictionary& _dictionary; + const ISearchContext& _baseSearchCtx; + const BitVector* _bv; // bitvector if _useBitVector has been set + const FrozenDictionary _frozenDictionary; + DictionaryConstIterator _lowerDictItr; + DictionaryConstIterator _upperDictItr; + uint64_t _numValues; // attr.getStatus().getNumValues(); + uint32_t _uniqueValues; + uint32_t _docIdLimit; + uint32_t _dictSize; + EntryRef _pidx; + EntryRef _frozenRoot; // Posting list in tree form + float _FSTC; // Filtering Search Time Constant + float _PLSTC; // Posting List Search Time Constant + bool _hasWeight; + bool _useBitVector; + mutable std::optional<size_t> _counted_hits; // Snapshot of size of posting lists in range PostingListSearchContext(const IEnumStoreDictionary& dictionary, bool has_btree_dictionary, uint32_t docIdLimit, uint64_t numValues, bool hasWeight, bool useBitVector, const ISearchContext &baseSearchCtx); @@ -89,6 +97,18 @@ protected: return (numHits > 1000) && (calculateFilteringCost() < calculatePostingListCost(numHits)); } + virtual bool use_posting_list_when_non_strict(const queryeval::ExecuteInfo&) const { + return false; + } + virtual bool fallback_to_approx_num_hits() const { + return ((_uniqueValues > MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION) && + ((_uniqueValues * MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > static_cast<int>(_docIdLimit)) || + (calculateApproxNumHits() * MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > _docIdLimit) || + (_uniqueValues > MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION*10))); + } + virtual size_t countHits() const = 0; + virtual void fillArray() = 0; + virtual void fillBitVector() = 0; }; @@ -110,19 +130,15 @@ protected: */ PostingListMerger<DataT> _merger; - static const long MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION = 100; - static const long MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 20; - static const long MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 10; - PostingListSearchContextT(const IEnumStoreDictionary& dictionary, uint32_t docIdLimit, uint64_t numValues, bool hasWeight, const PostingList &postingList, bool useBitVector, const ISearchContext &baseSearchCtx); ~PostingListSearchContextT() override; void lookupSingle(); - size_t countHits() const; - void fillArray(); - void fillBitVector(); + size_t countHits() const override; + void fillArray() override; + void fillBitVector() override; void fetchPostings(const queryeval::ExecuteInfo & strict) override; // this will be called instead of the fetchPostings function in some cases @@ -141,22 +157,41 @@ protected: template <class DataT> class PostingListFoldedSearchContextT : public PostingListSearchContextT<DataT> { +public: + static constexpr uint32_t MAX_POSTING_INDEXES_SIZE = 10000; + protected: using Parent = PostingListSearchContextT<DataT>; using Dictionary = typename Parent::Dictionary; + using DictionaryConstIterator = Dictionary::ConstIterator; + using EntryRef = vespalib::datastore::EntryRef; using PostingList = typename Parent::PostingList; + using Parent::_counted_hits; + using Parent::_docIdLimit; using Parent::_lowerDictItr; - using Parent::_uniqueValues; + using Parent::_merger; using Parent::_postingList; - using Parent::_docIdLimit; - using Parent::countHits; + using Parent::_uniqueValues; + using Parent::_upperDictItr; using Parent::singleHits; + using Parent::use_dictionary_entry; + + mutable DictionaryConstIterator _resume_scan_itr; + mutable std::vector<EntryRef> _posting_indexes; PostingListFoldedSearchContextT(const IEnumStoreDictionary& dictionary, uint32_t docIdLimit, uint64_t numValues, bool hasWeight, const PostingList &postingList, bool useBitVector, const ISearchContext &baseSearchCtx); - - unsigned int approximateHits() const override; + ~PostingListFoldedSearchContextT() override; + + bool fallback_to_approx_num_hits() const override; + size_t countHits() const override; + template <bool fill_array> + void fill_array_or_bitvector_helper(EntryRef pidx); + template <bool fill_array> + void fill_array_or_bitvector(); + void fillArray() override; + void fillBitVector() override; }; @@ -188,6 +223,7 @@ private: bool use_single_dictionary_entry(PostingListSearchContext::DictionaryConstIterator it) const { return use_dictionary_entry(it); } + bool use_posting_list_when_non_strict(const queryeval::ExecuteInfo&) const override; public: StringPostingSearchContext(BaseSC&& base_sc, bool useBitVector, const AttrT &toBeSearched); }; @@ -320,13 +356,45 @@ StringPostingSearchContext<BaseSC, AttrT, DataT>::use_dictionary_entry(PostingLi ++it; return false; } else if (this->isFuzzy()) { - if (this->getFuzzyMatcher().isMatch(_enumStore.get_value(it.getKey().load_acquire()))) { + return this->is_fuzzy_match(_enumStore.get_value(it.getKey().load_acquire()), it, _enumStore.get_data_store()); + } + return true; +} + +template <typename BaseSC, typename AttrT, typename DataT> +bool +StringPostingSearchContext<BaseSC, AttrT, DataT>::use_posting_list_when_non_strict(const queryeval::ExecuteInfo& info) const +{ + if (this->isFuzzy()) { + uint32_t exp_doc_hits = this->_docIdLimit * info.hitRate(); + constexpr uint32_t fuzzy_use_posting_list_doc_limit = 10000; + /** + * The above constant was derived after a query latency experiment with fuzzy matching + * on 2M documents with a dictionary size of 292070. + * + * Cost per document in dfa-based fuzzy matching (scanning the dictionary and merging posting lists) - strict iterator: + * 2.8 ms / 2k = 0.0014 ms + * 4.4 ms / 20k = 0.00022 ms + * 9.0 ms / 200k = 0.000045 ms + * 98 ms / 1M = 0.000098 ms + * + * Cost per document in lookup-based fuzzy matching - non-strict iterator: + * 7.6 ms / 2k = 0.0038 ms + * 54 ms / 20k = 0.0027 ms + * 529 ms / 200k = 0.0026 ms + * + * Based on this experiment, we observe that we should avoid lookup-based fuzzy matching + * when the number of documents to calculate this on exceeds a number between 2000 - 20000. + * + * Also note that the cost of scanning the dictionary and performing the fuzzy matching + * is already performed at this point. + * The only work remaining if returning true is merging the posting lists. + */ + if (exp_doc_hits > fuzzy_use_posting_list_doc_limit) { return true; } - ++it; - return false; } - return true; + return false; } template <typename BaseSC, typename AttrT, typename DataT> diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index bd1cc1191a7..e10cc8a6f41 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -9,7 +9,6 @@ #include "postingstore.hpp" #include "posting_list_traverser.h" #include <vespa/searchlib/queryeval/emptysearch.h> -#include <vespa/searchlib/queryeval/executeinfo.h> #include <vespa/searchlib/common/bitvectoriterator.h> #include <vespa/searchlib/common/growablebitvector.h> @@ -58,57 +57,48 @@ PostingListSearchContextT<DataT>::lookupSingle() } } - template <typename DataT> size_t PostingListSearchContextT<DataT>::countHits() const { + if (_counted_hits.has_value()) { + return _counted_hits.value(); + } size_t sum(0); - for (auto it(_lowerDictItr); it != _upperDictItr;) { - if (use_dictionary_entry(it)) { - sum += _postingList.frozenSize(it.getData().load_acquire()); - ++it; - } + for (auto it(_lowerDictItr); it != _upperDictItr; ++it) { + sum += _postingList.frozenSize(it.getData().load_acquire()); } + _counted_hits = sum; return sum; } - template <typename DataT> void PostingListSearchContextT<DataT>::fillArray() { - for (auto it(_lowerDictItr); it != _upperDictItr;) { - if (use_dictionary_entry(it)) { - _merger.addToArray(PostingListTraverser<PostingList>(_postingList, - it.getData().load_acquire())); - ++it; - } + for (auto it(_lowerDictItr); it != _upperDictItr; ++it) { + _merger.addToArray(PostingListTraverser<PostingList>(_postingList, + it.getData().load_acquire())); } _merger.merge(); } - template <typename DataT> void PostingListSearchContextT<DataT>::fillBitVector() { - for (auto it(_lowerDictItr); it != _upperDictItr;) { - if (use_dictionary_entry(it)) { - _merger.addToBitVector(PostingListTraverser<PostingList>(_postingList, - it.getData().load_acquire())); - ++it; - } + for (auto it(_lowerDictItr); it != _upperDictItr; ++it) { + _merger.addToBitVector(PostingListTraverser<PostingList>(_postingList, + it.getData().load_acquire())); } } - template <typename DataT> void PostingListSearchContextT<DataT>::fetchPostings(const queryeval::ExecuteInfo & execInfo) { if (!_merger.merge_done() && _uniqueValues >= 2u) { - if (execInfo.isStrict() && !fallbackToFiltering()) { + if ((execInfo.isStrict() || use_posting_list_when_non_strict(execInfo)) && !fallbackToFiltering()) { size_t sum(countHits()); if (sum < _docIdLimit / 64) { _merger.reserveArray(_uniqueValues, sum); @@ -227,29 +217,20 @@ template <typename DataT> unsigned int PostingListSearchContextT<DataT>::approximateHits() const { - unsigned int numHits = 0; + size_t numHits = 0; if (_uniqueValues == 0u) { } else if (_uniqueValues == 1u) { numHits = singleHits(); } else { if (this->fallbackToFiltering()) { numHits = _docIdLimit; - } else if (_uniqueValues > MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION) { - if ((_uniqueValues * MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > static_cast<int>(_docIdLimit)) || - (this->calculateApproxNumHits() * MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > _docIdLimit) || - (_uniqueValues > MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION*10)) - { - numHits = this->calculateApproxNumHits(); - } else { - // XXX: Unsafe - numHits = countHits(); - } + } else if (this->fallback_to_approx_num_hits()) { + numHits = this->calculateApproxNumHits(); } else { - // XXX: Unsafe numHits = countHits(); } } - return numHits; + return std::min(numHits, size_t(std::numeric_limits<uint32_t>::max())); } @@ -282,28 +263,98 @@ PostingListFoldedSearchContextT<DataT>:: PostingListFoldedSearchContextT(const IEnumStoreDictionary& dictionary, uint32_t docIdLimit, uint64_t numValues, bool hasWeight, const PostingList &postingList, bool useBitVector, const ISearchContext &searchContext) - : Parent(dictionary, docIdLimit, numValues, hasWeight, postingList, useBitVector, searchContext) + : Parent(dictionary, docIdLimit, numValues, hasWeight, postingList, useBitVector, searchContext), + _resume_scan_itr(), + _posting_indexes() { } +template <typename DataT> +PostingListFoldedSearchContextT<DataT>::~PostingListFoldedSearchContextT() = default; template <typename DataT> -unsigned int -PostingListFoldedSearchContextT<DataT>::approximateHits() const +bool +PostingListFoldedSearchContextT<DataT>::fallback_to_approx_num_hits() const { - unsigned int numHits = 0; - if (_uniqueValues == 0u) { - } else if (_uniqueValues == 1u) { - numHits = singleHits(); + return false; +} + +template <typename DataT> +size_t +PostingListFoldedSearchContextT<DataT>::countHits() const +{ + if (_counted_hits.has_value()) { + return _counted_hits.value(); + } + size_t sum(0); + bool overflow = false; + for (auto it(_lowerDictItr); it != _upperDictItr;) { + if (use_dictionary_entry(it)) { + auto pidx = it.getData().load_acquire(); + if (pidx.valid()) { + sum += _postingList.frozenSize(pidx); + if (!overflow) { + if (_posting_indexes.size() < MAX_POSTING_INDEXES_SIZE) { + _posting_indexes.emplace_back(pidx); + } else { + overflow = true; + _resume_scan_itr = it; + } + } + } + ++it; + } + } + _counted_hits = sum; + return sum; +} + +template <typename DataT> +template <bool fill_array> +void +PostingListFoldedSearchContextT<DataT>::fill_array_or_bitvector_helper(EntryRef pidx) +{ + if constexpr (fill_array) { + _merger.addToArray(PostingListTraverser<PostingList>(_postingList, pidx)); } else { - if (this->fallbackToFiltering()) { - numHits = _docIdLimit; - } else { - // XXX: Unsafe - numHits = countHits(); + _merger.addToBitVector(PostingListTraverser<PostingList>(_postingList, pidx)); + } +} + +template <typename DataT> +template <bool fill_array> +void +PostingListFoldedSearchContextT<DataT>::fill_array_or_bitvector() +{ + for (auto pidx : _posting_indexes) { + fill_array_or_bitvector_helper<fill_array>(pidx); + } + if (_resume_scan_itr.valid()) { + for (auto it(_resume_scan_itr); it != _upperDictItr;) { + if (use_dictionary_entry(it)) { + auto pidx = it.getData().load_acquire(); + if (pidx.valid()) { + fill_array_or_bitvector_helper<fill_array>(pidx); + } + ++it; + } } } - return numHits; + _merger.merge(); +} + +template <typename DataT> +void +PostingListFoldedSearchContextT<DataT>::fillArray() +{ + fill_array_or_bitvector<true>(); +} + +template <typename DataT> +void +PostingListFoldedSearchContextT<DataT>::fillBitVector() +{ + fill_array_or_bitvector<false>(); } } diff --git a/searchlib/src/vespa/searchlib/attribute/string_matcher.h b/searchlib/src/vespa/searchlib/attribute/string_matcher.h index 05089e1251a..09ba813cefe 100644 --- a/searchlib/src/vespa/searchlib/attribute/string_matcher.h +++ b/searchlib/src/vespa/searchlib/attribute/string_matcher.h @@ -32,6 +32,11 @@ protected: const vespalib::Regex& getRegex() const { return _helper.getRegex(); } const vespalib::FuzzyMatcher& getFuzzyMatcher() const { return _helper.getFuzzyMatcher(); } const QueryTermUCS4* get_query_term_ptr() const noexcept { return _query_term.get(); } + + template <typename DictionaryConstIteratorType> + bool is_fuzzy_match(const char* word, DictionaryConstIteratorType& itr, const DfaStringComparator::DataStoreType& data_store) const { + return _helper.is_fuzzy_match(word, itr, data_store); + } }; } diff --git a/searchlib/src/vespa/searchlib/attribute/string_search_helper.cpp b/searchlib/src/vespa/searchlib/attribute/string_search_helper.cpp index 1efe39667b8..82709997228 100644 --- a/searchlib/src/vespa/searchlib/attribute/string_search_helper.cpp +++ b/searchlib/src/vespa/searchlib/attribute/string_search_helper.cpp @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "string_search_helper.h" +#include "dfa_fuzzy_matcher.h" +#include "i_enum_store_dictionary.h" #include <vespa/searchlib/query/query_term_ucs4.h> #include <vespa/vespalib/text/lowercase.h> #include <vespa/vespalib/text/utf8.h> @@ -9,9 +11,32 @@ namespace search::attribute { +using FMA = vespalib::FuzzyMatchingAlgorithm; +using LDT = vespalib::fuzzy::LevenshteinDfa::DfaType; + +namespace { + +LDT +to_dfa_type(FMA algorithm) +{ + switch (algorithm) { + case FMA::DfaImplicit: + return LDT::Implicit; + case FMA::DfaExplicit: + return LDT::Explicit; + case FMA::DfaTable: + return LDT::Table; + default: + return LDT::Implicit; + } +} + +} + StringSearchHelper::StringSearchHelper(QueryTermUCS4 & term, bool cased, vespalib::FuzzyMatchingAlgorithm fuzzy_matching_algorithm) : _regex(), _fuzzyMatcher(), + _dfa_fuzzy_matcher(), _term(), _termLen(), _isPrefix(term.isPrefix()), @@ -24,12 +49,18 @@ StringSearchHelper::StringSearchHelper(QueryTermUCS4 & term, bool cased, vespali ? vespalib::Regex::from_pattern(term.getTerm(), vespalib::Regex::Options::None) : vespalib::Regex::from_pattern(term.getTerm(), vespalib::Regex::Options::IgnoreCase); } else if (isFuzzy()) { - (void) fuzzy_matching_algorithm; - // TODO: Select implementation based on algorithm. _fuzzyMatcher = std::make_unique<vespalib::FuzzyMatcher>(term.getTerm(), term.getFuzzyMaxEditDistance(), term.getFuzzyPrefixLength(), isCased()); + if ((fuzzy_matching_algorithm != FMA::BruteForce) && + (term.getFuzzyMaxEditDistance() <= 2)) { + _dfa_fuzzy_matcher = std::make_unique<DfaFuzzyMatcher>(term.getTerm(), + term.getFuzzyMaxEditDistance(), + term.getFuzzyPrefixLength(), + isCased(), + to_dfa_type(fuzzy_matching_algorithm)); + } } else if (isCased()) { _term = term.getTerm(); _termLen = strlen(_term); @@ -48,7 +79,7 @@ StringSearchHelper::isMatch(const char *src) const noexcept { return getRegex().valid() && getRegex().partial_match(std::string_view(src)); } if (__builtin_expect(isFuzzy(), false)) { - return getFuzzyMatcher().isMatch(src); + return _dfa_fuzzy_matcher ? _dfa_fuzzy_matcher->is_match(src) : getFuzzyMatcher().isMatch(src); } if (__builtin_expect(isCased(), false)) { int res = strncmp(_term, src, _termLen); @@ -67,4 +98,27 @@ StringSearchHelper::isMatch(const char *src) const noexcept { return (_ucs4[j] == 0 && (val == 0 || isPrefix())); } +template <typename DictionaryConstIteratorType> +bool +StringSearchHelper::is_fuzzy_match(const char* word, DictionaryConstIteratorType& itr, const DfaStringComparator::DataStoreType& data_store) const +{ + if (_dfa_fuzzy_matcher) { + return _dfa_fuzzy_matcher->is_match(word, itr, data_store); + } else { + if (_fuzzyMatcher->isMatch(word)) { + return true; + } + ++itr; + return false; + } +} + +template +bool +StringSearchHelper::is_fuzzy_match(const char*, EnumPostingTree::ConstIterator&, const DfaStringComparator::DataStoreType&) const; + +template +bool +StringSearchHelper::is_fuzzy_match(const char*, EnumTree::ConstIterator&, const DfaStringComparator::DataStoreType&) const; + } diff --git a/searchlib/src/vespa/searchlib/attribute/string_search_helper.h b/searchlib/src/vespa/searchlib/attribute/string_search_helper.h index 0e7a116a874..e59291e24a7 100644 --- a/searchlib/src/vespa/searchlib/attribute/string_search_helper.h +++ b/searchlib/src/vespa/searchlib/attribute/string_search_helper.h @@ -2,6 +2,7 @@ #pragma once +#include "dfa_string_comparator.h" #include <vespa/vespalib/fuzzy/fuzzy_matching_algorithm.h> #include <vespa/vespalib/regex/regex.h> @@ -10,6 +11,8 @@ namespace search { class QueryTermUCS4; } namespace search::attribute { +class DfaFuzzyMatcher; + /** * Helper class for search context when scanning string fields * It handles different search settings like prefix, regex and cased/uncased. @@ -29,11 +32,16 @@ public: bool isCased() const noexcept { return _isCased; } bool isFuzzy() const noexcept { return _isFuzzy; } const vespalib::Regex & getRegex() const noexcept { return _regex; } - const FuzzyMatcher & getFuzzyMatcher() const noexcept { return *_fuzzyMatcher; } + const FuzzyMatcher& getFuzzyMatcher() const noexcept { return *_fuzzyMatcher; } + + template <typename DictionaryConstIteratorType> + bool is_fuzzy_match(const char* word, DictionaryConstIteratorType& itr, const DfaStringComparator::DataStoreType& data_store) const; + private: using ucs4_t = uint32_t; vespalib::Regex _regex; std::unique_ptr<FuzzyMatcher> _fuzzyMatcher; + std::unique_ptr<DfaFuzzyMatcher> _dfa_fuzzy_matcher; std::unique_ptr<ucs4_t[]> _ucs4; const char * _term; uint32_t _termLen; // measured in bytes diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h index 134c4215fc2..17b4c2d09ba 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h @@ -34,17 +34,17 @@ public: uint64_t _subqueries; }; private: - bool isRawScore() const { return _flags & RAW_SCORE_FLAG; } - bool isMultiPos() const { return _flags & MULTIPOS_FLAG; } - bool empty() const { return _sz == 0; } - void clear() { _sz = 0; } - bool allocated() const { return isMultiPos(); } - const TermFieldMatchDataPosition * getFixed() const { return reinterpret_cast<const TermFieldMatchDataPosition *>(_data._position); } - TermFieldMatchDataPosition * getFixed() { return reinterpret_cast<TermFieldMatchDataPosition *>(_data._position); } - const TermFieldMatchDataPosition * getMultiple() const { return _data._positions._positions; } - TermFieldMatchDataPosition * getMultiple() { return _data._positions._positions; } - int32_t getElementWeight() const { return empty() ? 1 : allocated() ? getMultiple()->getElementWeight() : getFixed()->getElementWeight(); } - uint32_t getMaxElementLength() const { return empty() ? 0 : allocated() ? _data._positions._maxElementLength : getFixed()->getElementLen(); } + bool isRawScore() const noexcept { return _flags & RAW_SCORE_FLAG; } + bool isMultiPos() const noexcept { return _flags & MULTIPOS_FLAG; } + bool empty() const noexcept { return _sz == 0; } + void clear() noexcept { _sz = 0; } + bool allocated() const noexcept { return isMultiPos(); } + const TermFieldMatchDataPosition * getFixed() const noexcept { return reinterpret_cast<const TermFieldMatchDataPosition *>(_data._position); } + TermFieldMatchDataPosition * getFixed() noexcept { return reinterpret_cast<TermFieldMatchDataPosition *>(_data._position); } + const TermFieldMatchDataPosition * getMultiple() const noexcept { return _data._positions._positions; } + TermFieldMatchDataPosition * getMultiple() noexcept { return _data._positions._positions; } + int32_t getElementWeight() const noexcept { return empty() ? 1 : allocated() ? getMultiple()->getElementWeight() : getFixed()->getElementWeight(); } + uint32_t getMaxElementLength() const noexcept { return empty() ? 0 : allocated() ? _data._positions._maxElementLength : getFixed()->getElementLen(); } void appendPositionToAllocatedVector(const TermFieldMatchDataPosition &pos); void allocateVector(); void resizePositionVector(size_t sz) __attribute__((noinline)); @@ -70,8 +70,8 @@ private: public: PositionsIterator begin() const { return allocated() ? getMultiple() : getFixed(); } PositionsIterator end() const { return allocated() ? getMultiple() + _sz : empty() ? getFixed() : getFixed()+1; } - size_t size() const { return _sz; } - size_t capacity() const { return allocated() ? _data._positions._allocated : 1; } + size_t size() const noexcept { return _sz; } + size_t capacity() const noexcept { return allocated() ? _data._positions._allocated : 1; } void reservePositions(size_t sz) { if (sz > capacity()) { if (!allocated()) { @@ -114,7 +114,7 @@ public: * * @return field id **/ - uint32_t getFieldId() const { + uint32_t getFieldId() const noexcept { return __builtin_expect(_fieldId != ILLEGAL_FIELD_ID, true) ? _fieldId : IllegalFieldId; } @@ -125,7 +125,7 @@ public: * @return this object (for chaining) * @param docId id of the document we are generating match information for **/ - TermFieldMatchData &reset(uint32_t docId) { + TermFieldMatchData &reset(uint32_t docId) noexcept { _docId = docId; _sz = 0; _numOccs = 0; @@ -145,7 +145,7 @@ public: * @return this object (for chaining) * @param docId id of the document we are generating match information for **/ - TermFieldMatchData &resetOnlyDocId(uint32_t docId) { + TermFieldMatchData &resetOnlyDocId(uint32_t docId) noexcept { _docId = docId; return *this; } @@ -160,13 +160,13 @@ public: * @param docId id of the document we have matched * @param score a raw score for the matched document **/ - TermFieldMatchData &setRawScore(uint32_t docId, feature_t score) { + TermFieldMatchData &setRawScore(uint32_t docId, feature_t score) noexcept { resetOnlyDocId(docId); enableRawScore(); _data._rawScore = score; return *this; } - TermFieldMatchData & enableRawScore() { + TermFieldMatchData & enableRawScore() noexcept { _flags |= RAW_SCORE_FLAG; return *this; } @@ -176,16 +176,16 @@ public: * * @return raw score **/ - feature_t getRawScore() const { + feature_t getRawScore() const noexcept { return __builtin_expect(isRawScore(), true) ? _data._rawScore : 0.0; } - void setSubqueries(uint32_t docId, uint64_t subqueries) { + void setSubqueries(uint32_t docId, uint64_t subqueries) noexcept { resetOnlyDocId(docId); _data._subqueries = subqueries; } - uint64_t getSubqueries() const { + uint64_t getSubqueries() const noexcept { if (!empty() || isRawScore()) { return 0; } @@ -197,7 +197,7 @@ public: * * @return document id **/ - uint32_t getDocId() const { + uint32_t getDocId() const noexcept { return _docId; } @@ -208,7 +208,7 @@ public: * * @return weight **/ - int32_t getWeight() const { + int32_t getWeight() const noexcept { if (__builtin_expect(_sz == 0, false)) { return 1; } @@ -246,8 +246,8 @@ public: return FieldPositionsIterator(len != 0 ? len : FieldPositionsIterator::UNKNOWN_LENGTH, begin(), end()); } - uint16_t getNumOccs() const { return _numOccs; } - uint16_t getFieldLength() const { return _fieldLength; } + uint16_t getNumOccs() const noexcept { return _numOccs; } + uint16_t getFieldLength() const noexcept { return _fieldLength; } void setNumOccs(uint16_t value) { _numOccs = value; } void setFieldLength(uint16_t value) { _fieldLength = value; } @@ -256,23 +256,25 @@ public: * This indicates if this instance is actually used for ranking or not. * @return true if it is not needed. */ - bool isNotNeeded() const { return ((_flags & (UNPACK_NORMAL_FEATURES_FLAG | UNPACK_INTERLEAVED_FEATURES_FLAG)) == 0u); } + bool isNotNeeded() const noexcept { + return ((_flags & (UNPACK_NORMAL_FEATURES_FLAG | UNPACK_INTERLEAVED_FEATURES_FLAG)) == 0u); + } - bool needs_normal_features() const { return ((_flags & UNPACK_NORMAL_FEATURES_FLAG) != 0u); } + bool needs_normal_features() const noexcept { return ((_flags & UNPACK_NORMAL_FEATURES_FLAG) != 0u); } - bool needs_interleaved_features() const { return ((_flags & UNPACK_INTERLEAVED_FEATURES_FLAG) != 0u); } + bool needs_interleaved_features() const noexcept{ return ((_flags & UNPACK_INTERLEAVED_FEATURES_FLAG) != 0u); } /** * Tag that this instance is not really used for ranking. */ - void tagAsNotNeeded() { + void tagAsNotNeeded() noexcept { _flags &= ~(UNPACK_NORMAL_FEATURES_FLAG | UNPACK_INTERLEAVED_FEATURES_FLAG); } /** * Tag that this instance is used for ranking (normal features) */ - void setNeedNormalFeatures(bool needed) { + void setNeedNormalFeatures(bool needed) noexcept { if (needed) { _flags |= UNPACK_NORMAL_FEATURES_FLAG; } else { @@ -283,7 +285,7 @@ public: /** * Tag that this instance is used for ranking (interleaved features) */ - void setNeedInterleavedFeatures(bool needed) { + void setNeedInterleavedFeatures(bool needed) noexcept { if (needed) { _flags |= UNPACK_INTERLEAVED_FEATURES_FLAG; } else { @@ -297,7 +299,7 @@ public: * * @return constant **/ - static uint32_t invalidId() { return 0xdeadbeefU; } + static uint32_t invalidId() noexcept { return 0xdeadbeefU; } }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 3f6085ef7ff..94d1a4917fd 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -621,7 +621,7 @@ IntermediateBlueprint::fetchPostings(const ExecuteInfo &execInfo) double nextHitRate = execInfo.hitRate(); for (size_t i = 0; i < _children.size(); ++i) { Blueprint & child = *_children[i]; - child.fetchPostings(ExecuteInfo::create(execInfo.isStrict() && inheritStrict(i), nextHitRate)); + child.fetchPostings(ExecuteInfo::create(execInfo.isStrict() && inheritStrict(i), nextHitRate, execInfo.getDoom())); nextHitRate = computeNextHitRate(child, nextHitRate); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp index 795f5f1424a..4322cafb5c8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp @@ -66,7 +66,7 @@ DotProductBlueprint::createFilterSearch(bool strict, FilterConstraint constraint void DotProductBlueprint::fetchPostings(const ExecuteInfo &execInfo) { - ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo.hitRate()); + ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo); for (size_t i = 0; i < _terms.size(); ++i) { _terms[i]->fetchPostings(childInfo); } diff --git a/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp b/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp index 604e20d2262..e5d20f047f5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp @@ -4,17 +4,7 @@ namespace search::queryeval { -const ExecuteInfo ExecuteInfo::TRUE(true, 1.0); -const ExecuteInfo ExecuteInfo::FALSE(false, 1.0); - -ExecuteInfo -ExecuteInfo::create(bool strict) { - return create(strict, 1.0); -} - -ExecuteInfo -ExecuteInfo::create(bool strict, double hitRate) { - return ExecuteInfo(strict, hitRate); -} +const ExecuteInfo ExecuteInfo::TRUE(true, 1.0, nullptr); +const ExecuteInfo ExecuteInfo::FALSE(false, 1.0, nullptr); } diff --git a/searchlib/src/vespa/searchlib/queryeval/executeinfo.h b/searchlib/src/vespa/searchlib/queryeval/executeinfo.h index e161b2bdab7..2dd34284bef 100644 --- a/searchlib/src/vespa/searchlib/queryeval/executeinfo.h +++ b/searchlib/src/vespa/searchlib/queryeval/executeinfo.h @@ -1,8 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright 2019 Oath inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/vespalib/util/doom.h> + namespace search::queryeval { /** @@ -11,20 +12,37 @@ namespace search::queryeval { */ class ExecuteInfo { public: - ExecuteInfo() : ExecuteInfo(false, 1.0) { } - bool isStrict() const { return _strict; } - double hitRate() const { return _hitRate; } + ExecuteInfo() noexcept : ExecuteInfo(false, 1.0F, nullptr) { } + bool isStrict() const noexcept { return _strict; } + float hitRate() const noexcept { return _hitRate; } + bool soft_doom() const noexcept { return _doom && _doom->soft_doom(); } + const vespalib::Doom * getDoom() const { return _doom; } static const ExecuteInfo TRUE; static const ExecuteInfo FALSE; - static ExecuteInfo create(bool strict); - static ExecuteInfo create(bool strict, double HitRate); + static ExecuteInfo create(bool strict, const ExecuteInfo & org) noexcept { + return {strict, org._hitRate, org.getDoom()}; + } + static ExecuteInfo create(bool strict, const vespalib::Doom * doom) noexcept { + return create(strict, 1.0F, doom); + } + static ExecuteInfo create(bool strict, float hitRate, const vespalib::Doom * doom) noexcept { + return {strict, hitRate, doom}; + } + static ExecuteInfo create(bool strict) noexcept { + return create(strict, 1.0F); + } + static ExecuteInfo create(bool strict, float hitRate) noexcept { + return create(strict, hitRate, nullptr); + } private: - ExecuteInfo(bool strict, double hitRate_in) - : _hitRate(hitRate_in), + ExecuteInfo(bool strict, float hitRate_in, const vespalib::Doom * doom) noexcept + : _doom(doom), + _hitRate(hitRate_in), _strict(strict) { } - double _hitRate; - bool _strict; + const vespalib::Doom * _doom; + float _hitRate; + bool _strict; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/iterator_pack.cpp b/searchlib/src/vespa/searchlib/queryeval/iterator_pack.cpp index e9f1b526a87..5280100b5bc 100644 --- a/searchlib/src/vespa/searchlib/queryeval/iterator_pack.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/iterator_pack.cpp @@ -28,6 +28,7 @@ SearchIteratorPack::SearchIteratorPack(const std::vector<SearchIterator*> &child _children.emplace_back(child); } assert((_children.size() == _childMatch.size()) || _childMatch.empty()); + assert(_children.size() < 0x10000); } SearchIteratorPack::SearchIteratorPack(const std::vector<SearchIterator*> &children, MatchDataUP md) @@ -36,7 +37,6 @@ SearchIteratorPack::SearchIteratorPack(const std::vector<SearchIterator*> &child std::unique_ptr<BitVector> SearchIteratorPack::get_hits(uint32_t begin_id, uint32_t end_id) const { - BitVector::UP result = TermwiseHelper::orChildren(_children.begin(), _children.end(), begin_id); if (! result ) { result = BitVector::create(begin_id, end_id); diff --git a/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h b/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h index 907371008d9..ca1e6461533 100644 --- a/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h +++ b/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h @@ -31,27 +31,25 @@ public: // TODO: use MultiSearch::Children to pass ownership SearchIteratorPack(const std::vector<SearchIterator*> &children, MatchDataUP md); - uint32_t get_docid(uint32_t ref) const { + uint32_t get_docid(uint16_t ref) const { return _children[ref]->getDocId(); } - uint32_t seek(uint32_t ref, uint32_t docid) { + uint32_t seek(uint16_t ref, uint32_t docid) { _children[ref]->seek(docid); return _children[ref]->getDocId(); } - int32_t get_weight(uint32_t ref, uint32_t docid) { + int32_t get_weight(uint16_t ref, uint32_t docid) { _children[ref]->doUnpack(docid); return _childMatch[ref]->getWeight(); } - void unpack(uint32_t ref, uint32_t docid) { + void unpack(uint16_t ref, uint32_t docid) { _children[ref]->doUnpack(docid); } - size_t size() const { - return _children.size(); - } + uint16_t size() const { return _children.size(); } void initRange(uint32_t begin, uint32_t end) { for (auto & child: _children) { child->initRange(begin, end); diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp index 8b8db0f293a..4eb57eda911 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp @@ -11,51 +11,13 @@ namespace search::queryeval { using vespalib::Trinary; using vespalib::hwaccelrated::IAccelrated; +using Meta = MultiBitVectorBase::Meta; namespace { -template<typename Update> -class MultiBitVectorIterator : public MultiBitVectorIteratorBase -{ -public: - explicit MultiBitVectorIterator(Children children) - : MultiBitVectorIteratorBase(std::move(children)), - _update(), - _accel(IAccelrated::getAccelerator()), - _lastWords() - { - static_assert(sizeof(_lastWords) == 64, "Lastwords should have 64 byte size"); - static_assert(NumWordsInBatch == 8, "Batch size should be 8 words."); - memset(_lastWords, 0, sizeof(_lastWords)); - } -protected: - void updateLastValue(uint32_t docId) noexcept; - void strictSeek(uint32_t docId) noexcept; -private: - void doSeek(uint32_t docId) override; - Trinary is_strict() const override { return Trinary::False; } - bool acceptExtraFilter() const noexcept final { return Update::isAnd(); } - Update _update; - const IAccelrated & _accel; - alignas(64) Word _lastWords[8]; - static constexpr size_t NumWordsInBatch = sizeof(_lastWords) / sizeof(Word); -}; - -template<typename Update> -class MultiBitVectorIteratorStrict final : public MultiBitVectorIterator<Update> -{ -public: - explicit MultiBitVectorIteratorStrict(MultiSearch::Children children) - : MultiBitVectorIterator<Update>(std::move(children)) - { } -private: - void doSeek(uint32_t docId) override { this->strictSeek(docId); } - Trinary is_strict() const override { return Trinary::True; } -}; - struct And { using Word = BitWord::Word; - void operator () (const IAccelrated & accel, size_t offset, const std::vector<std::pair<const void *, bool>> & src, void *dest) noexcept { + void operator () (const IAccelrated & accel, size_t offset, const std::vector<Meta> & src, void *dest) noexcept { accel.and64(offset, src, dest); } static bool isAnd() noexcept { return true; } @@ -63,19 +25,49 @@ struct And { struct Or { using Word = BitWord::Word; - void operator () (const IAccelrated & accel, size_t offset, const std::vector<std::pair<const void *, bool>> & src, void *dest) noexcept { + void operator () (const IAccelrated & accel, size_t offset, const std::vector<Meta> & src, void *dest) noexcept { accel.or64(offset, src, dest); } static bool isAnd() noexcept { return false; } }; +} + +MultiBitVectorBase::MultiBitVectorBase(size_t reserved) + : _numDocs(std::numeric_limits<uint32_t>::max()), + _lastMaxDocIdLimit(0), + _lastMaxDocIdLimitRequireFetch(0), + _lastValue(0), + _bvs() +{ + _bvs.reserve(reserved); +} + +void +MultiBitVectorBase::addBitVector(Meta bv, uint32_t docIdLimit) { + _numDocs = std::min(_numDocs, docIdLimit); + _bvs.push_back(bv); +} + +template <typename Update> +MultiBitVector<Update>::MultiBitVector(size_t reserved) + : MultiBitVectorBase(reserved), + _update(), + _accel(IAccelrated::getAccelerator()), + _lastWords() +{ + static_assert(sizeof(_lastWords) == 64, "Lastwords should have 64 byte size"); + static_assert(NumWordsInBatch == 8, "Batch size should be 8 words."); + memset(_lastWords, 0, sizeof(_lastWords)); +} + template<typename Update> -void MultiBitVectorIterator<Update>::updateLastValue(uint32_t docId) noexcept +bool +MultiBitVector<Update>::updateLastValue(uint32_t docId) noexcept { if (docId >= _lastMaxDocIdLimit) { if (__builtin_expect(docId >= _numDocs, false)) { - setAtEnd(); - return; + return true; } const uint32_t index(BitWord::wordNum(docId)); if (docId >= _lastMaxDocIdLimitRequireFetch) { @@ -86,37 +78,104 @@ void MultiBitVectorIterator<Update>::updateLastValue(uint32_t docId) noexcept _lastValue = _lastWords[index % NumWordsInBatch]; _lastMaxDocIdLimit = (index + 1) * BitWord::WordLen; } + return false; } template<typename Update> -void -MultiBitVectorIterator<Update>::doSeek(uint32_t docId) +uint32_t +MultiBitVector<Update>::strictSeek(uint32_t docId) noexcept +{ + bool atEnd; + for (atEnd = updateLastValue(docId), _lastValue = _lastValue & BitWord::checkTab(docId); + (_lastValue == 0) && __builtin_expect(! atEnd, true); + atEnd = updateLastValue(_lastMaxDocIdLimit)); + if (__builtin_expect(!atEnd, true)) { + return _lastMaxDocIdLimit - BitWord::WordLen + vespalib::Optimized::lsbIdx(_lastValue); + } + return _numDocs; +} + +template<typename Update> +bool +MultiBitVector<Update>::seek(uint32_t docId) noexcept { - updateLastValue(docId); - if (__builtin_expect( ! isAtEnd(), true)) { + bool atEnd = updateLastValue(docId); + if (__builtin_expect( ! atEnd, true)) { if (_lastValue & BitWord::mask(docId)) { - setDocId(docId); + return true; } } + return false; } +namespace { + template<typename Update> -void -MultiBitVectorIterator<Update>::strictSeek(uint32_t docId) noexcept +class MultiBitVectorIterator : public MultiBitVectorIteratorBase { - for (updateLastValue(docId), _lastValue = _lastValue & BitWord::checkTab(docId); - (_lastValue == 0) && __builtin_expect(! isAtEnd(), true); - updateLastValue(_lastMaxDocIdLimit)); - if (__builtin_expect(!isAtEnd(), true)) { - docId = _lastMaxDocIdLimit - BitWord::WordLen + vespalib::Optimized::lsbIdx(_lastValue); - if (__builtin_expect(docId >= _numDocs, false)) { - setAtEnd(); +public: + explicit MultiBitVectorIterator(Children children) + : MultiBitVectorIteratorBase(std::move(children)), + _mbv(getChildren().size() + 1) + { + for (const auto & child : getChildren()) { + const auto * bv = static_cast<const BitVectorIterator *>(child.get()); + _mbv.addBitVector(Meta(bv->getBitValues(), bv->isInverted()), bv->getDocIdLimit()); + } + } + void initRange(uint32_t beginId, uint32_t endId) override { + MultiBitVectorIteratorBase::initRange(beginId, endId); + _mbv.reset(); + } + UP andWith(UP filter, uint32_t estimate) override; +protected: + void doSeek(uint32_t docId) override; + Trinary is_strict() const override { return Trinary::False; } + bool acceptExtraFilter() const noexcept final { return _mbv.acceptExtraFilter(); } + MultiBitVector<Update> _mbv; +}; + +template<typename Update> +class MultiBitVectorIteratorStrict final : public MultiBitVectorIterator<Update> +{ +public: + explicit MultiBitVectorIteratorStrict(MultiSearch::Children children) + : MultiBitVectorIterator<Update>(std::move(children)) + { } +private: + void doSeek(uint32_t docId) override { + docId = this->_mbv.strictSeek(docId); + if (__builtin_expect(docId >= this->getEndId(), false)) { + this->setAtEnd(); } else { - setDocId(docId); + this->setDocId(docId); } } + Trinary is_strict() const override { return Trinary::True; } +}; + +template<typename Update> +void +MultiBitVectorIterator<Update>::doSeek(uint32_t docId) +{ + if (_mbv.seek(docId)) { + setDocId(docId); + } } +template <typename Update> +SearchIterator::UP +MultiBitVectorIterator<Update>::andWith(UP filter, uint32_t estimate) +{ + (void) estimate; + if (filter->isBitVector() && acceptExtraFilter()) { + const auto & bv = static_cast<const BitVectorIterator &>(*filter); + _mbv.addBitVector(Meta(bv.getBitValues(), bv.isInverted()), bv.getDocIdLimit()); + insert(getChildren().size(), std::move(filter)); + _mbv.reset(); + } + return filter; +} using AndBVIterator = MultiBitVectorIterator<And>; using AndBVIteratorStrict = MultiBitVectorIteratorStrict<And>; @@ -148,20 +207,8 @@ bool canOptimize(const MultiSearch & s) { } MultiBitVectorIteratorBase::MultiBitVectorIteratorBase(Children children) : - MultiSearch(std::move(children)), - _numDocs(std::numeric_limits<unsigned int>::max()), - _lastMaxDocIdLimit(0), - _lastMaxDocIdLimitRequireFetch(0), - _lastValue(0), - _bvs() -{ - _bvs.reserve(getChildren().size()); - for (const auto & child : getChildren()) { - const auto * bv = static_cast<const BitVectorIterator *>(child.get()); - _bvs.emplace_back(bv->getBitValues(), bv->isInverted()); - _numDocs = std::min(_numDocs, bv->getDocIdLimit()); - } -} + MultiSearch(std::move(children)) +{ } MultiBitVectorIteratorBase::~MultiBitVectorIteratorBase() = default; @@ -169,22 +216,6 @@ void MultiBitVectorIteratorBase::initRange(uint32_t beginId, uint32_t endId) { MultiSearch::initRange(beginId, endId); - _lastMaxDocIdLimit = 0; - _lastMaxDocIdLimitRequireFetch = 0; -} - -SearchIterator::UP -MultiBitVectorIteratorBase::andWith(UP filter, uint32_t estimate) -{ - (void) estimate; - if (filter->isBitVector() && acceptExtraFilter()) { - const auto & bv = static_cast<const BitVectorIterator &>(*filter); - _bvs.emplace_back(bv.getBitValues(), bv.isInverted()); - insert(getChildren().size(), std::move(filter)); - _lastMaxDocIdLimit = 0; // force reload - _lastMaxDocIdLimitRequireFetch = 0; - } - return filter; } void diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h index d99e439af0b..5c3d17c6786 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h @@ -6,8 +6,45 @@ #include "unpackinfo.h" #include <vespa/searchlib/common/bitword.h> +namespace vespalib::hwaccelrated { class IAccelrated; } + namespace search::queryeval { +class MultiBitVectorBase { +public: + using Meta = std::pair<const void *, bool>; + MultiBitVectorBase(size_t reserved); + using Word = BitWord::Word; + void reset() { + _lastMaxDocIdLimit = 0; + _lastMaxDocIdLimitRequireFetch = 0; + } + void addBitVector(Meta bv, uint32_t docIdLimit); +protected: + uint32_t _numDocs; + uint32_t _lastMaxDocIdLimit; // next documentid requiring recomputation. + uint32_t _lastMaxDocIdLimitRequireFetch; + Word _lastValue; // Last value computed + std::vector<Meta> _bvs; +}; + +template <typename Update> +class MultiBitVector : public MultiBitVectorBase { +public: + explicit MultiBitVector(size_t reserved); + uint32_t strictSeek(uint32_t docId) noexcept; + bool seek(uint32_t docId) noexcept; + bool acceptExtraFilter() const noexcept { return Update::isAnd(); } +private: + bool updateLastValue(uint32_t docId) noexcept; + using IAccelrated = vespalib::hwaccelrated::IAccelrated; + + Update _update; + const IAccelrated & _accel; + alignas(64) Word _lastWords[8]; + static constexpr size_t NumWordsInBatch = sizeof(_lastWords) / sizeof(Word); +}; + class MultiBitVectorIteratorBase : public MultiSearch { public: @@ -20,18 +57,9 @@ public: */ static SearchIterator::UP optimize(SearchIterator::UP parent); protected: - using Word = BitWord::Word; - MultiBitVectorIteratorBase(Children children); - using MetaWord = std::pair<const void *, bool>; - - uint32_t _numDocs; - uint32_t _lastMaxDocIdLimit; // next documentid requiring recomputation. - uint32_t _lastMaxDocIdLimitRequireFetch; - Word _lastValue; // Last value computed - std::vector<MetaWord> _bvs; + explicit MultiBitVectorIteratorBase(Children children); private: virtual bool acceptExtraFilter() const noexcept = 0; - UP andWith(UP filter, uint32_t estimate) override; void doUnpack(uint32_t docid) override; static SearchIterator::UP optimizeMultiSearch(SearchIterator::UP parent); diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp index 9c3910b20f9..16461487525 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp @@ -57,7 +57,7 @@ void SameElementBlueprint::fetchPostings(const ExecuteInfo &execInfo) { for (size_t i = 0; i < _terms.size(); ++i) { - _terms[i]->fetchPostings(ExecuteInfo::create(execInfo.isStrict() && (i == 0), execInfo.hitRate())); + _terms[i]->fetchPostings(ExecuteInfo::create(execInfo.isStrict() && (i == 0), execInfo)); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp index 48a09f099a6..eb6241a99d5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp @@ -4,7 +4,6 @@ #include "wand_parts.h" #include "parallel_weak_and_search.h" #include <vespa/searchlib/queryeval/field_spec.hpp> -#include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/visit.hpp> @@ -77,10 +76,10 @@ ParallelWeakAndBlueprint::createLeafSearch(const search::fef::TermFieldMatchData const State &childState = _terms[i]->getState(); assert(childState.numFields() == 1); // TODO: pass ownership with unique_ptr - terms.push_back(wand::Term(_terms[i]->createSearch(*childrenMatchData, true).release(), - _weights[i], - childState.estimate().estHits, - childState.field(0).resolve(*childrenMatchData))); + terms.emplace_back(_terms[i]->createSearch(*childrenMatchData, true).release(), + _weights[i], + childState.estimate().estHits, + childState.field(0).resolve(*childrenMatchData)); } return SearchIterator::UP (ParallelWeakAndSearch::create(terms, @@ -101,9 +100,9 @@ ParallelWeakAndBlueprint::createFilterSearch(bool strict, FilterConstraint const void ParallelWeakAndBlueprint::fetchPostings(const ExecuteInfo & execInfo) { - ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo.hitRate()); - for (size_t i = 0; i < _terms.size(); ++i) { - _terms[i]->fetchPostings(childInfo); + ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo); + for (const auto & _term : _terms) { + _term->fetchPostings(childInfo); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp index 4e06f170253..55009e714b9 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp @@ -33,10 +33,8 @@ WeightedSetTermMatchingElementsSearch::WeightedSetTermMatchingElementsSearch(con _search() { _tfmda.add(&_tfmd); - auto generic_search = bp.createLeafSearch(_tfmda, false); - auto weighted_set_term_search = dynamic_cast<WeightedSetTermSearch *>(generic_search.get()); - generic_search.release(); - _search.reset(weighted_set_term_search); + _search.reset(static_cast<WeightedSetTermSearch *>(bp.createLeafSearch(_tfmda, false).release())); + } WeightedSetTermMatchingElementsSearch::~WeightedSetTermMatchingElementsSearch() = default; @@ -94,7 +92,6 @@ WeightedSetTermBlueprint::addTerm(Blueprint::UP term, int32_t weight, HitEstimat _terms.push_back(std::move(term)); } - SearchIterator::UP WeightedSetTermBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool) const { @@ -120,16 +117,16 @@ WeightedSetTermBlueprint::create_matching_elements_search(const MatchingElements if (fields.has_field(_children_field.getName())) { return std::make_unique<WeightedSetTermMatchingElementsSearch>(*this, _children_field.getName(), _terms); } else { - return std::unique_ptr<MatchingElementsSearch>(); + return {}; } } void WeightedSetTermBlueprint::fetchPostings(const ExecuteInfo &execInfo) { - ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo.hitRate()); - for (size_t i = 0; i < _terms.size(); ++i) { - _terms[i]->fetchPostings(childInfo); + ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo); + for (const auto & _term : _terms) { + _term->fetchPostings(childInfo); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h index 0e3c82444d7..9c8d6d88329 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h @@ -18,7 +18,7 @@ class WeightedSetTermBlueprint : public ComplexLeafBlueprint std::vector<Blueprint::UP> _terms; public: - WeightedSetTermBlueprint(const FieldSpec &field); + explicit WeightedSetTermBlueprint(const FieldSpec &field); WeightedSetTermBlueprint(const WeightedSetTermBlueprint &) = delete; WeightedSetTermBlueprint &operator=(const WeightedSetTermBlueprint &) = delete; ~WeightedSetTermBlueprint() override; diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp index ee3978705cf..9568c02cf32 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp @@ -2,6 +2,7 @@ #include "weighted_set_term_search.h" #include <vespa/searchlib/common/bitvector.h> +#include <vespa/searchlib/attribute/document_weight_or_filter_search.h> #include <vespa/vespalib/objects/visit.h> #include <vespa/searchcommon/attribute/i_search_context.h> @@ -21,7 +22,7 @@ private: struct CmpDocId { const uint32_t *termPos; - CmpDocId(const uint32_t *tp) : termPos(tp) {} + explicit CmpDocId(const uint32_t *tp) : termPos(tp) {} bool operator()(const ref_t &a, const ref_t &b) const { return (termPos[a] < termPos[b]); } @@ -29,7 +30,7 @@ private: struct CmpWeight { const int32_t *weight; - CmpWeight(const int32_t *w) : weight(w) {} + explicit CmpWeight(const int32_t *w) : weight(w) {} bool operator()(const ref_t &a, const ref_t &b) const { return (weight[a] > weight[b]); } @@ -45,7 +46,7 @@ private: ref_t *_data_stash; ref_t *_data_end; IteratorPack _children; - bool _field_is_filter; + bool _need_match_data; void seek_child(ref_t child, uint32_t docId) { _termPos[child] = _children.seek(child, docId); @@ -61,7 +62,7 @@ private: } public: - WeightedSetTermSearchImpl(search::fef::TermFieldMatchData &tmd, + WeightedSetTermSearchImpl(fef::TermFieldMatchData &tmd, bool field_is_filter, const std::vector<int32_t> &weights, IteratorPack &&iteratorPack) @@ -75,7 +76,7 @@ public: _data_stash(nullptr), _data_end(nullptr), _children(std::move(iteratorPack)), - _field_is_filter(field_is_filter) + _need_match_data(!field_is_filter && !_tmd.isNotNeeded()) { HEAP::require_left_heap(); assert(_children.size() > 0); @@ -86,7 +87,7 @@ public: } _data_begin = &_data_space[0]; _data_end = _data_begin + _data_space.size(); - if (!_field_is_filter && !_tmd.isNotNeeded()) { + if (_need_match_data) { _tmd.reservePositions(_children.size()); } } @@ -112,7 +113,7 @@ public: } void doUnpack(uint32_t docId) override { - if (!_field_is_filter && !_tmd.isNotNeeded()) { + if (_need_match_data) { _tmd.reset(docId); pop_matching_children(docId); std::sort(_data_stash, _data_end, _cmpWeight); @@ -171,6 +172,10 @@ WeightedSetTermSearch::create(const std::vector<SearchIterator *> &children, using ArrayHeapImpl = WeightedSetTermSearchImpl<vespalib::LeftArrayHeap, SearchIteratorPack>; using HeapImpl = WeightedSetTermSearchImpl<vespalib::LeftHeap, SearchIteratorPack>; + if (tmd.isNotNeeded()) { + return attribute::DocumentWeightOrFilterSearch::create(children, std::move(match_data)); + } + if (children.size() < 128) { return SearchIterator::UP(new ArrayHeapImpl(tmd, field_is_filter, weights, SearchIteratorPack(children, std::move(match_data)))); } @@ -180,7 +185,7 @@ WeightedSetTermSearch::create(const std::vector<SearchIterator *> &children, //----------------------------------------------------------------------------- SearchIterator::UP -WeightedSetTermSearch::create(search::fef::TermFieldMatchData &tmd, +WeightedSetTermSearch::create(fef::TermFieldMatchData &tmd, bool field_is_filter, const std::vector<int32_t> &weights, std::vector<DocumentWeightIterator> &&iterators) diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.h b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.h index e3e12c27f28..b30d3bc3301 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.h @@ -10,12 +10,9 @@ #include <memory> #include <vector> -namespace search { -namespace fef { -class TermFieldMatchData; -} // namespace fef +namespace search::fef { class TermFieldMatchData; } -namespace queryeval { +namespace search::queryeval { class Blueprint; @@ -26,7 +23,7 @@ class Blueprint; class WeightedSetTermSearch : public SearchIterator { protected: - WeightedSetTermSearch() {} + WeightedSetTermSearch() = default; public: // TODO: pass ownership with unique_ptr @@ -47,6 +44,4 @@ public: virtual void find_matching_elements(uint32_t docid, const std::vector<std::unique_ptr<Blueprint>> &child_blueprints, std::vector<uint32_t> &dst) = 0; }; -} // namespace search::queryeval -} // namespace search - +} diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 009f8170669..1ef670f96ac 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -14,7 +14,6 @@ vespa_add_library(storage_storageserver OBJECT documentapiconverter.cpp fnet_metrics_wrapper.cpp mergethrottler.cpp - opslogger.cpp priorityconverter.cpp rpcrequestwrapper.cpp service_layer_error_listener.cpp diff --git a/storage/src/vespa/storage/storageserver/distributornode.cpp b/storage/src/vespa/storage/storageserver/distributornode.cpp index ab80381f5d4..431dd89b613 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.cpp +++ b/storage/src/vespa/storage/storageserver/distributornode.cpp @@ -3,7 +3,6 @@ #include "distributornode.h" #include "bouncer.h" #include "communicationmanager.h" -#include "opslogger.h" #include "statemanager.h" #include <vespa/storage/common/hostreporter/hostinfo.h> #include <vespa/storage/common/i_storage_chain_builder.h> @@ -96,7 +95,6 @@ DistributorNode::createChain(IStorageChainBuilder &builder) std::unique_ptr<StateManager> stateManager(releaseStateManager()); builder.add(std::make_unique<Bouncer>(dcr, _configUri)); - builder.add(std::make_unique<OpsLogger>(dcr, _configUri)); // Distributor instance registers a host info reporter with the state // manager, which is safe since the lifetime of said state manager // extends to the end of the process. diff --git a/storage/src/vespa/storage/storageserver/opslogger.cpp b/storage/src/vespa/storage/storageserver/opslogger.cpp deleted file mode 100644 index dcf7ddf4a92..00000000000 --- a/storage/src/vespa/storage/storageserver/opslogger.cpp +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "opslogger.h" -#include <vespa/storageframework/generic/clock/clock.h> -#include <vespa/storageapi/message/persistence.h> -#include <vespa/config/helper/configfetcher.hpp> -#include <vespa/config/subscription/configuri.h> -#include <sstream> - -#include <vespa/log/log.h> -LOG_SETUP(".operationslogger"); - -namespace storage { - -OpsLogger::OpsLogger(StorageComponentRegister& compReg, - const config::ConfigUri & configUri) - : StorageLink("Operations logger"), - _lock(), - _fileName(), - _targetFile(nullptr), - _component(compReg, "opslogger"), - _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())) -{ - _configFetcher->subscribe<vespa::config::content::core::StorOpsloggerConfig>(configUri.getConfigId(), this); - _configFetcher->start(); -} - -OpsLogger::~OpsLogger() -{ - closeNextLink(); - LOG(debug, "Deleting link %s.", toString().c_str()); - - if (_targetFile) { - fclose(_targetFile); - } -} - -void -OpsLogger::onClose() -{ - // Avoid getting config during shutdown - _configFetcher->close(); -} - -void -OpsLogger::configure(std::unique_ptr<vespa::config::content::core::StorOpsloggerConfig> config) -{ - std::lock_guard lock(_lock); - // If no change in state, ignore - if (config->targetfile == _fileName) return; - // If a change we need to close old handle if open - if (_targetFile != nullptr) { - fclose(_targetFile); - _targetFile = nullptr; - } - // Set up the new operations log file - _fileName = config->targetfile; - if (_fileName.length() > 0) { - _targetFile = fopen(_fileName.c_str(), "a+b"); - - if (!_targetFile) { - LOG(warning, "Could not open file %s for operations logging", - _fileName.c_str()); - } - } -} - -void -OpsLogger::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "OpsLogger()"; -} - -bool -OpsLogger::onPutReply(const std::shared_ptr<api::PutReply>& msg) -{ - if (_targetFile == nullptr) return false; - std::ostringstream ost; - ost << vespalib::to_string(_component.getClock().getSystemTime()) - << "\tPUT\t" << msg->getDocumentId() << "\t" - << msg->getResult() << "\n"; - { - std::lock_guard lock(_lock); - if (_targetFile == nullptr) return false; - fwrite(ost.str().c_str(), ost.str().length(), 1, _targetFile); - fflush(_targetFile); - } - return false; -} - -bool -OpsLogger::onUpdateReply(const std::shared_ptr<api::UpdateReply>& msg) -{ - if (_targetFile == nullptr) return false; - std::ostringstream ost; - ost << vespalib::to_string(_component.getClock().getSystemTime()) - << "\tUPDATE\t" << msg->getDocumentId() << "\t" - << msg->getResult() << "\n"; - { - std::lock_guard lock(_lock); - if (_targetFile == nullptr) return false; - fwrite(ost.str().c_str(), ost.str().length(), 1, _targetFile); - fflush(_targetFile); - } - return false; -} - -bool -OpsLogger::onRemoveReply(const std::shared_ptr<api::RemoveReply>& msg) -{ - if (_targetFile == nullptr) return false; - std::ostringstream ost; - ost << vespalib::to_string(_component.getClock().getSystemTime()) - << "\tREMOVE\t" << msg->getDocumentId() << "\t" - << msg->getResult() << "\n"; - { - std::lock_guard lock(_lock); - if (_targetFile == nullptr) return false; - fwrite(ost.str().c_str(), ost.str().length(), 1, _targetFile); - fflush(_targetFile); - } - return false; -} - -bool -OpsLogger::onGetReply(const std::shared_ptr<api::GetReply>& msg) -{ - if (_targetFile == nullptr) return false; - std::ostringstream ost; - ost << vespalib::to_string(_component.getClock().getSystemTime()) - << "\tGET\t" << msg->getDocumentId() << "\t" - << msg->getResult() << "\n"; - { - std::lock_guard lock(_lock); - if (_targetFile == nullptr) return false; - fwrite(ost.str().c_str(), ost.str().length(), 1, _targetFile); - fflush(_targetFile); - } - return false; -} - -} // storage diff --git a/storage/src/vespa/storage/storageserver/opslogger.h b/storage/src/vespa/storage/storageserver/opslogger.h deleted file mode 100644 index 039cb72969e..00000000000 --- a/storage/src/vespa/storage/storageserver/opslogger.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -/** - * \class storage::OpsLogger - * - * \brief Storage link that can be configured to log all storage operations to - * a file. -*/ -#pragma once - -#include <vespa/storage/common/storagelink.h> -#include <vespa/storage/common/storagecomponent.h> -#include <vespa/storageapi/messageapi/storagemessage.h> -#include <vespa/storageapi/message/state.h> -#include <vespa/storage/config/config-stor-opslogger.h> -#include <vespa/config/helper/ifetchercallback.h> - -namespace config { - class ConfigUri; - class ConfigFetcher; -} - -namespace storage { - -class OpsLogger : public StorageLink, - public config::IFetcherCallback<vespa::config::content::core::StorOpsloggerConfig> { -public: - explicit OpsLogger(StorageComponentRegister&, - const config::ConfigUri & configUri); - ~OpsLogger() override; - - void onClose() override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - bool onPutReply(const std::shared_ptr<api::PutReply>& msg) override; - bool onUpdateReply(const std::shared_ptr<api::UpdateReply>& msg) override; - bool onRemoveReply(const std::shared_ptr<api::RemoveReply>& msg) override; - bool onGetReply(const std::shared_ptr<api::GetReply>& msg) override; - - /** Ignore all replies on the way down the storage chain. */ - bool onDown(const std::shared_ptr<api::StorageMessage>&) override { return false; }; - void configure(std::unique_ptr<vespa::config::content::core::StorOpsloggerConfig> config) override; -private: - std::mutex _lock; - std::string _fileName; - FILE * _targetFile; - framework::Component _component; - - std::unique_ptr<config::ConfigFetcher> _configFetcher; -}; - -} diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.cpp b/storage/src/vespa/storage/storageserver/servicelayernode.cpp index 65615bea2dd..846d6ed09bf 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.cpp +++ b/storage/src/vespa/storage/storageserver/servicelayernode.cpp @@ -5,7 +5,6 @@ #include "communicationmanager.h" #include "changedbucketownershiphandler.h" #include "mergethrottler.h" -#include "opslogger.h" #include "statemanager.h" #include "priorityconverter.h" #include "service_layer_error_listener.h" @@ -167,7 +166,6 @@ ServiceLayerNode::createChain(IStorageChainBuilder &builder) _communicationManager = communication_manager.get(); builder.add(std::move(communication_manager)); builder.add(std::make_unique<Bouncer>(compReg, _configUri)); - builder.add(std::make_unique<OpsLogger>(compReg, _configUri)); auto merge_throttler_up = std::make_unique<MergeThrottler>(_configUri, compReg); auto merge_throttler = merge_throttler_up.get(); builder.add(std::move(merge_throttler_up)); diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 2972ea7745e..3ae0923aef0 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -1,211 +1,206 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#[non-test] -# Contains dependencies that are not used exclusively in 'test' scope -ai.djl:api:0.23.0 ai.djl.huggingface:tokenizers:0.23.0 -aopalliance:aopalliance:1.0 +ai.djl:api:0.23.0 +aopalliance:aopalliance:${aopalliance.vespa.version} backport-util-concurrent:backport-util-concurrent:3.1 ch.qos.logback:logback-classic:1.2.10 ch.qos.logback:logback-core:1.2.10 classworlds:classworlds:1.1-alpha-2 -com.amazonaws:aws-java-sdk-core:1.12.540 -com.amazonaws:aws-java-sdk-kms:1.12.540 -com.amazonaws:aws-java-sdk-s3:1.12.540 -com.amazonaws:aws-java-sdk-ssm:1.12.540 -com.amazonaws:aws-java-sdk-sts:1.12.540 -com.amazonaws:jmespath-java:1.12.540 -com.auth0:java-jwt:4.4.0 -com.fasterxml.jackson.core:jackson-annotations:2.15.2 -com.fasterxml.jackson.core:jackson-core:2.15.2 -com.fasterxml.jackson.core:jackson-databind:2.15.2 -com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.15.2 -com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.15.2 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2 -com.github.luben:zstd-jni:1.5.5-5 +com.amazonaws:aws-java-sdk-core:${aws-sdk.vespa.version} +com.amazonaws:aws-java-sdk-kms:${aws-sdk.vespa.version} +com.amazonaws:aws-java-sdk-s3:${aws-sdk.vespa.version} +com.amazonaws:aws-java-sdk-ssm:${aws-sdk.vespa.version} +com.amazonaws:aws-java-sdk-sts:${aws-sdk.vespa.version} +com.amazonaws:jmespath-java:${aws-sdk.vespa.version} +com.auth0:java-jwt:${java-jwt.vespa.version} +com.fasterxml.jackson.core:jackson-annotations:${jackson2.vespa.version} +com.fasterxml.jackson.core:jackson-core:${jackson2.vespa.version} +com.fasterxml.jackson.core:jackson-databind:${jackson-databind.vespa.version} +com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson2.vespa.version} +com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jackson2.vespa.version} +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jackson2.vespa.version} +com.github.luben:zstd-jni:${luben.zstd.vespa.version} com.github.spotbugs:spotbugs-annotations:3.1.9 -com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.21.1 +com.google.code.findbugs:jsr305:${findbugs.vespa.version} +com.google.errorprone:error_prone_annotations:${error-prone-annotations.vespa.version} com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:32.1.2-jre -com.google.inject:guice:6.0.0 +com.google.guava:guava:${guava.vespa.version} +com.google.inject:guice:${guice.vespa.version} com.google.j2objc:j2objc-annotations:2.8 -com.google.protobuf:protobuf-java:3.24.3 -com.ibm.icu:icu4j:73.2 -com.microsoft.onnxruntime:onnxruntime:1.15.1 -com.sun.activation:javax.activation:1.2.0 +com.google.jimfs:jimfs:${jimfs.vespa.version} +com.google.protobuf:protobuf-java:${protobuf.vespa.version} +com.ibm.icu:icu4j:${icu4j.vespa.version} +com.microsoft.onnxruntime:onnxruntime:${onnxruntime.vespa.version} +com.sun.activation:javax.activation:${properties-maven-plugin.vespa.version} com.sun.istack:istack-commons-runtime:4.1.2 -com.sun.xml.bind:jaxb-core:2.3.0.1 -com.sun.xml.bind:jaxb-impl:2.3.0 +com.sun.xml.bind:jaxb-core:${jaxb-core.vespa.version} +com.sun.xml.bind:jaxb-impl:${jaxb-impl.vespa.version} com.thaiopensource:jing:20091111 -com.yahoo.athenz:athenz-auth-core:1.11.42 -com.yahoo.athenz:athenz-client-common:1.11.42 -com.yahoo.athenz:athenz-zms-core:1.11.42 -com.yahoo.athenz:athenz-zpe-java-client:1.11.42 -com.yahoo.athenz:athenz-zts-core:1.11.42 +com.yahoo.athenz:athenz-auth-core:${athenz.vespa.version} +com.yahoo.athenz:athenz-client-common:${athenz.vespa.version} +com.yahoo.athenz:athenz-zms-core:${athenz.vespa.version} +com.yahoo.athenz:athenz-zpe-java-client:${athenz.vespa.version} +com.yahoo.athenz:athenz-zts-core:${athenz.vespa.version} com.yahoo.rdl:rdl-java:1.5.4 commons-cli:commons-cli:1.5.0 -commons-codec:commons-codec:1.16.0 +commons-codec:commons-codec:${commons-codec.vespa.version} commons-fileupload:commons-fileupload:1.5 -commons-io:commons-io:2.13.0 -commons-logging:commons-logging:1.2 -io.airlift:airline:0.9 -io.dropwizard.metrics:metrics-core:4.2.19 -io.jsonwebtoken:jjwt-api:0.11.5 -io.jsonwebtoken:jjwt-impl:0.11.5 -io.jsonwebtoken:jjwt-jackson:0.11.5 -io.netty:netty-buffer:4.1.98.Final -io.netty:netty-codec:4.1.98.Final -io.netty:netty-common:4.1.98.Final -io.netty:netty-handler:4.1.98.Final -io.netty:netty-resolver:4.1.98.Final -io.netty:netty-tcnative:2.0.61.Final -io.netty:netty-tcnative-classes:2.0.61.Final -io.netty:netty-transport:4.1.98.Final -io.netty:netty-transport-classes-epoll:4.1.98.Final -io.netty:netty-transport-native-epoll:4.1.98.Final -io.netty:netty-transport-native-unix-common:4.1.98.Final -io.prometheus:simpleclient:0.16.0 -io.prometheus:simpleclient_common:0.16.0 -io.prometheus:simpleclient_tracer_common:0.16.0 -io.prometheus:simpleclient_tracer_otel:0.16.0 -io.prometheus:simpleclient_tracer_otel_agent:0.16.0 -jakarta.inject:jakarta.inject-api:2.0.1 -javax.activation:javax.activation-api:1.2.0 -javax.annotation:javax.annotation-api:1.2 -javax.inject:javax.inject:1 -javax.servlet:javax.servlet-api:3.1.0 -javax.ws.rs:javax.ws.rs-api:2.1.1 -javax.xml.bind:jaxb-api:2.3.1 -joda-time:joda-time:2.12.5 -junit:junit:4.13.2 -net.java.dev.jna:jna:5.13.0 -net.openhft:zero-allocation-hashing:0.16 -org.antlr:antlr-runtime:3.5.3 -org.antlr:antlr4-runtime:4.13.1 -org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:1.3.6 -org.apache.commons:commons-compress:1.24.0 -org.apache.commons:commons-csv:1.10.0 -org.apache.commons:commons-exec:1.3 -org.apache.commons:commons-lang3:3.13.0 -org.apache.commons:commons-math3:3.6.1 -org.apache.curator:curator-client:5.5.0 -org.apache.curator:curator-framework:5.5.0 -org.apache.curator:curator-recipes:5.5.0 -org.apache.felix:org.apache.felix.framework:7.0.5 -org.apache.felix:org.apache.felix.log:1.3.0 -org.apache.httpcomponents:httpclient:4.5.14 -org.apache.httpcomponents:httpcore:4.4.16 -org.apache.httpcomponents:httpmime:4.5.14 -org.apache.httpcomponents.client5:httpclient5:5.2.1 -org.apache.httpcomponents.core5:httpcore5:5.2.3 -org.apache.httpcomponents.core5:httpcore5-h2:5.2.3 -org.apache.lucene:lucene-analysis-common:9.7.0 -org.apache.lucene:lucene-core:9.7.0 -org.apache.maven:maven-archiver:3.6.1 -org.apache.maven:maven-artifact:3.9.4 +commons-io:commons-io:${commons-io.vespa.version} +commons-logging:commons-logging:${commons-logging.vespa.version} +io.airlift:airline:${airline.vespa.version} +io.dropwizard.metrics:metrics-core:${dropwizard.metrics.vespa.version} +io.jsonwebtoken:jjwt-api:${java-jjwt.vespa.version} +io.jsonwebtoken:jjwt-impl:${java-jjwt.vespa.version} +io.jsonwebtoken:jjwt-jackson:${java-jjwt.vespa.version} +io.netty:netty-buffer:${netty.vespa.version} +io.netty:netty-codec:${netty.vespa.version} +io.netty:netty-common:${netty.vespa.version} +io.netty:netty-handler:${netty.vespa.version} +io.netty:netty-resolver:${netty.vespa.version} +io.netty:netty-tcnative-classes:${netty-tcnative.vespa.version} +io.netty:netty-tcnative:${netty-tcnative.vespa.version} +io.netty:netty-transport-classes-epoll:${netty.vespa.version} +io.netty:netty-transport-native-epoll:${netty.vespa.version} +io.netty:netty-transport-native-unix-common:${netty.vespa.version} +io.netty:netty-transport:${netty.vespa.version} +io.prometheus:simpleclient:${prometheus.client.vespa.version} +io.prometheus:simpleclient_common:${prometheus.client.vespa.version} +io.prometheus:simpleclient_tracer_common:${prometheus.client.vespa.version} +io.prometheus:simpleclient_tracer_otel:${prometheus.client.vespa.version} +io.prometheus:simpleclient_tracer_otel_agent:${prometheus.client.vespa.version} +jakarta.inject:jakarta.inject-api:${jakarta.inject.vespa.version} +javax.activation:javax.activation-api:${properties-maven-plugin.vespa.version} +javax.annotation:javax.annotation-api:${commons-logging.vespa.version} +javax.inject:javax.inject:${javax.inject.vespa.version} +javax.servlet:javax.servlet-api:${javax.servlet-api.vespa.version} +javax.ws.rs:javax.ws.rs-api:${javax.ws.rs-api.vespa.version} +javax.xml.bind:jaxb-api:${jaxb-api.vespa.version} +joda-time:joda-time:${joda-time.vespa.version} +junit:junit:${junit4.vespa.version} +net.bytebuddy:byte-buddy-agent:${byte-buddy.vespa.version} +net.bytebuddy:byte-buddy:${byte-buddy.vespa.version} +net.java.dev.jna:jna:${jna.vespa.version} +net.openhft:zero-allocation-hashing:${zero-allocation-hashing.vespa.version} +org.antlr:antlr-runtime:${antlr.vespa.version} +org.antlr:antlr4-runtime:${antlr4.vespa.version} +org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:${spifly.vespa.version} +org.apache.commons:commons-compress:${commons-compress.vespa.version} +org.apache.commons:commons-csv:${commons-csv.vespa.version} +org.apache.commons:commons-exec:${commons-exec.vespa.version} +org.apache.commons:commons-lang3:${commons-lang3.vespa.version} +org.apache.commons:commons-math3:${commons.math3.vespa.version} +org.apache.curator:curator-client:${curator.vespa.version} +org.apache.curator:curator-framework:${curator.vespa.version} +org.apache.curator:curator-recipes:${curator.vespa.version} +org.apache.curator:curator-test:${curator.vespa.version} +org.apache.felix:org.apache.felix.framework:${felix.vespa.version} +org.apache.felix:org.apache.felix.log:${felix.log.vespa.version} +org.apache.httpcomponents.client5:httpclient5:${apache.httpclient5.vespa.version} +org.apache.httpcomponents.core5:httpcore5-h2:${apache.httpcore5.vespa.version} +org.apache.httpcomponents.core5:httpcore5:${apache.httpcore5.vespa.version} +org.apache.httpcomponents:httpclient:${apache.httpclient.vespa.version} +org.apache.httpcomponents:httpcore:${apache.httpcore.vespa.version} +org.apache.httpcomponents:httpmime:${apache.httpclient.vespa.version} +org.apache.lucene:lucene-analysis-common:${lucene.vespa.version} +org.apache.lucene:lucene-core:${lucene.vespa.version} +org.apache.maven.plugin-tools:maven-plugin-annotations:${maven-plugin-tools.vespa.version} +org.apache.maven.plugins:maven-jar-plugin:${maven-jar-plugin.vespa.version} +org.apache.maven.shared:file-management:3.1.0 +org.apache.maven.wagon:wagon-provider-api:${maven-wagon.vespa.version} +org.apache.maven:maven-archiver:${maven-archiver.vespa.version} org.apache.maven:maven-artifact-manager:2.2.1 -org.apache.maven:maven-model:3.9.4 -org.apache.maven:maven-plugin-api:3.9.4 +org.apache.maven:maven-artifact:${maven-core.vespa.version} +org.apache.maven:maven-model:${maven-core.vespa.version} +org.apache.maven:maven-plugin-api:${maven-core.vespa.version} org.apache.maven:maven-plugin-registry:2.2.1 org.apache.maven:maven-profile:2.2.1 org.apache.maven:maven-project:2.2.1 -org.apache.maven:maven-repository-metadata:3.9.4 -org.apache.maven:maven-settings:3.9.4 -org.apache.maven.plugin-tools:maven-plugin-annotations:3.9.0 -org.apache.maven.plugins:maven-jar-plugin:3.3.0 -org.apache.maven.shared:file-management:3.1.0 -org.apache.maven.wagon:wagon-provider-api:3.5.3 -org.apache.opennlp:opennlp-tools:2.3.0 +org.apache.maven:maven-repository-metadata:${maven-core.vespa.version} +org.apache.maven:maven-settings:${maven-core.vespa.version} +org.apache.opennlp:opennlp-tools:${opennlp.vespa.version} org.apache.velocity:velocity-engine-core:2.3 org.apache.yetus:audience-annotations:0.12.0 -org.apache.zookeeper:zookeeper:3.8.0 -org.apache.zookeeper:zookeeper:3.8.1 -org.apache.zookeeper:zookeeper-jute:3.8.0 +org.apache.zookeeper:zookeeper-jute:${zookeeper.client.vespa.version} org.apache.zookeeper:zookeeper-jute:3.8.1 -org.apiguardian:apiguardian-api:1.1.2 -org.bouncycastle:bcpkix-jdk18on:1.76 -org.bouncycastle:bcprov-jdk18on:1.76 -org.bouncycastle:bcutil-jdk18on:1.76 +org.apache.zookeeper:zookeeper:${zookeeper.client.vespa.version} +org.apache.zookeeper:zookeeper:3.8.1 +org.apiguardian:apiguardian-api:${apiguardian.vespa.version} +org.assertj:assertj-core:${assertj.vespa.version} +org.bouncycastle:bcpkix-jdk18on:${bouncycastle.vespa.version} +org.bouncycastle:bcprov-jdk18on:${bouncycastle.vespa.version} +org.bouncycastle:bcutil-jdk18on:${bouncycastle.vespa.version} org.codehaus.plexus:plexus-archiver:4.8.0 org.codehaus.plexus:plexus-classworlds:2.7.0 org.codehaus.plexus:plexus-component-annotations:1.5.5 org.codehaus.plexus:plexus-container-default:1.0-alpha-9-stable-1 org.codehaus.plexus:plexus-interpolation:1.26 -org.codehaus.plexus:plexus-io:3.4.1 -org.codehaus.plexus:plexus-utils:3.5.1 -org.eclipse.angus:angus-activation:2.0.1 -org.eclipse.collections:eclipse-collections:11.1.0 -org.eclipse.collections:eclipse-collections-api:11.1.0 -org.eclipse.jetty:jetty-alpn-client:11.0.16 -org.eclipse.jetty:jetty-alpn-java-client:11.0.16 -org.eclipse.jetty:jetty-alpn-java-server:11.0.16 -org.eclipse.jetty:jetty-alpn-server:11.0.16 -org.eclipse.jetty:jetty-client:11.0.16 -org.eclipse.jetty:jetty-http:11.0.16 -org.eclipse.jetty:jetty-io:11.0.16 -org.eclipse.jetty:jetty-jmx:11.0.16 -org.eclipse.jetty:jetty-security:11.0.16 -org.eclipse.jetty:jetty-server:11.0.16 -org.eclipse.jetty:jetty-servlet:11.0.16 -org.eclipse.jetty:jetty-util:11.0.16 -org.eclipse.jetty.http2:http2-client:11.0.16 -org.eclipse.jetty.http2:http2-common:11.0.16 -org.eclipse.jetty.http2:http2-hpack:11.0.16 -org.eclipse.jetty.http2:http2-http-client-transport:11.0.16 -org.eclipse.jetty.http2:http2-server:11.0.16 -org.eclipse.jetty.toolchain:jetty-jakarta-servlet-api:5.0.2 +org.codehaus.plexus:plexus-io:${maven-enforcer-plugin.vespa.version} +org.codehaus.plexus:plexus-utils:${maven-shade-plugin.vespa.version} +org.eclipse.angus:angus-activation:${jakarta.inject.vespa.version} +org.eclipse.collections:eclipse-collections-api:${eclipse-collections.vespa.version} +org.eclipse.collections:eclipse-collections:${eclipse-collections.vespa.version} +org.eclipse.jetty.http2:http2-client:${jetty.vespa.version} +org.eclipse.jetty.http2:http2-common:${jetty.vespa.version} +org.eclipse.jetty.http2:http2-hpack:${jetty.vespa.version} +org.eclipse.jetty.http2:http2-http-client-transport:${jetty.vespa.version} +org.eclipse.jetty.http2:http2-server:${jetty.vespa.version} +org.eclipse.jetty.toolchain:jetty-jakarta-servlet-api:${jetty-servlet-api.vespa.version} +org.eclipse.jetty:jetty-alpn-client:${jetty.vespa.version} +org.eclipse.jetty:jetty-alpn-java-client:${jetty.vespa.version} +org.eclipse.jetty:jetty-alpn-java-server:${jetty.vespa.version} +org.eclipse.jetty:jetty-alpn-server:${jetty.vespa.version} +org.eclipse.jetty:jetty-client:${jetty.vespa.version} +org.eclipse.jetty:jetty-http:${jetty.vespa.version} +org.eclipse.jetty:jetty-io:${jetty.vespa.version} +org.eclipse.jetty:jetty-jmx:${jetty.vespa.version} +org.eclipse.jetty:jetty-security:${jetty.vespa.version} +org.eclipse.jetty:jetty-server:${jetty.vespa.version} +org.eclipse.jetty:jetty-servlet:${jetty.vespa.version} +org.eclipse.jetty:jetty-util:${jetty.vespa.version} org.eclipse.sisu:org.eclipse.sisu.inject:0.3.5 org.eclipse.sisu:org.eclipse.sisu.plexus:0.3.5 org.fusesource.jansi:jansi:1.18 -org.glassfish.jaxb:jaxb-core:4.0.3 -org.glassfish.jaxb:jaxb-runtime:4.0.3 -org.glassfish.jaxb:txw2:4.0.3 -org.hamcrest:hamcrest:2.2 -org.hamcrest:hamcrest-core:2.2 -org.hdrhistogram:HdrHistogram:2.1.12 +org.glassfish.jaxb:jaxb-core:${jaxb.runtime.vespa.version} +org.glassfish.jaxb:jaxb-runtime:${jaxb.runtime.vespa.version} +org.glassfish.jaxb:txw2:${jaxb.runtime.vespa.version} +org.hamcrest:hamcrest-core:${hamcrest.vespa.version} +org.hamcrest:hamcrest:${hamcrest.vespa.version} +org.hdrhistogram:HdrHistogram:${hdrhistogram.vespa.version} org.iq80.snappy:snappy:0.4 -org.json:json:20230618 -org.junit.jupiter:junit-jupiter-api:5.10.0 -org.junit.jupiter:junit-jupiter-api:5.8.1 -org.junit.jupiter:junit-jupiter-engine:5.8.1 -org.junit.platform:junit-platform-commons:1.8.1 -org.junit.platform:junit-platform-engine:1.8.1 -org.junit.platform:junit-platform-launcher:1.8.1 +org.json:json:${org.json.vespa.version} +org.junit.jupiter:junit-jupiter-api:${junit.vespa.tenant.version} +org.junit.jupiter:junit-jupiter-api:${junit.vespa.version} +org.junit.jupiter:junit-jupiter-engine:${junit.vespa.tenant.version} +org.junit.jupiter:junit-jupiter-engine:${junit.vespa.version} +org.junit.jupiter:junit-jupiter-params:${junit.vespa.version} +org.junit.jupiter:junit-jupiter:${junit.vespa.version} +org.junit.platform:junit-platform-commons:${junit.platform.vespa.tenant.version} +org.junit.platform:junit-platform-commons:${junit.platform.vespa.version} +org.junit.platform:junit-platform-engine:${junit.platform.vespa.tenant.version} +org.junit.platform:junit-platform-engine:${junit.platform.vespa.version} +org.junit.platform:junit-platform-launcher:${junit.platform.vespa.tenant.version} +org.junit.vintage:junit-vintage-engine:${junit.vespa.tenant.version} +org.junit.vintage:junit-vintage-engine:${junit.vespa.version} org.kohsuke:libpam4j:1.11 -org.lz4:lz4-java:1.8.0 -org.opentest4j:opentest4j:1.3.0 -org.ow2.asm:asm:9.5 -org.ow2.asm:asm-analysis:9.5 -org.ow2.asm:asm-commons:9.5 -org.ow2.asm:asm-tree:9.5 -org.ow2.asm:asm-util:9.5 -org.questdb:questdb:7.3.2 -org.slf4j:jcl-over-slf4j:1.7.36 -org.slf4j:log4j-over-slf4j:1.7.36 -org.slf4j:slf4j-api:1.7.36 -org.slf4j:slf4j-jdk14:1.7.36 -org.slf4j:slf4j-simple:1.7.36 +org.lz4:lz4-java:${org.lz4.vespa.version} +org.mockito:mockito-core:${mockito.vespa.version} +org.mockito:mockito-junit-jupiter:${mockito.vespa.version} +org.objenesis:objenesis:3.3 +org.opentest4j:opentest4j:${opentest4j.vespa.version} +org.ow2.asm:asm-analysis:${asm.vespa.version} +org.ow2.asm:asm-commons:${asm.vespa.version} +org.ow2.asm:asm-tree:${asm.vespa.version} +org.ow2.asm:asm-util:${asm.vespa.version} +org.ow2.asm:asm:${asm.vespa.version} +org.questdb:questdb:${questdb.vespa.version} +org.slf4j:jcl-over-slf4j:${slf4j.vespa.version} +org.slf4j:log4j-over-slf4j:${slf4j.vespa.version} +org.slf4j:slf4j-api:${slf4j.vespa.version} +org.slf4j:slf4j-jdk14:${slf4j.vespa.version} +org.slf4j:slf4j-simple:${slf4j.vespa.version} org.tukaani:xz:1.9 -org.xerial.snappy:snappy-java:1.1.10.3 +org.wiremock:wiremock-standalone:${wiremock.vespa.version} +org.xerial.snappy:snappy-java:${snappy.vespa.version} software.amazon.ion:ion-java:1.0.2 -xerces:xercesImpl:2.12.2 - -#[test-only] -# Contains dependencies that are used exclusively in 'test' scope -com.google.jimfs:jimfs:1.3.0 -net.bytebuddy:byte-buddy:1.14.8 -net.bytebuddy:byte-buddy-agent:1.14.8 -org.apache.curator:curator-test:5.5.0 -org.assertj:assertj-core:3.24.2 -org.junit.jupiter:junit-jupiter:5.10.0 -org.junit.jupiter:junit-jupiter-engine:5.10.0 -org.junit.jupiter:junit-jupiter-params:5.10.0 -org.junit.platform:junit-platform-commons:1.10.0 -org.junit.platform:junit-platform-engine:1.10.0 -org.junit.vintage:junit-vintage-engine:5.10.0 -org.junit.vintage:junit-vintage-engine:5.8.1 -org.mockito:mockito-core:5.5.0 -org.mockito:mockito-junit-jupiter:5.5.0 -org.objenesis:objenesis:3.3 -org.wiremock:wiremock-standalone:3.1.0 +xerces:xercesImpl:${xerces.vespa.version} diff --git a/vespa-dependencies-enforcer/pom.xml b/vespa-dependencies-enforcer/pom.xml index 768e5708ee5..91820fd292b 100644 --- a/vespa-dependencies-enforcer/pom.xml +++ b/vespa-dependencies-enforcer/pom.xml @@ -38,7 +38,7 @@ </goals> <configuration> <rules> - <enforceDependencies implementation="com.yahoo.vespa.maven.plugin.enforcer.EnforceDependenciesAllProjects"> + <enforceDependencies implementation="com.yahoo.vespa.maven.plugin.enforcer.AllowedDependencies"> <rootProjectId>com.yahoo.vespa:vespa</rootProjectId> <specFile>allowed-maven-dependencies.txt</specFile> <ignored> @@ -47,14 +47,6 @@ <i>com.yahoo.vespa.bundle-plugin:*:*</i> <i>com.yahoo.vespa.jdisc_core:*:*</i> </ignored> - - <!-- Classifly all dependencies of below modules as 'test' --> - <testUtilProjects> - <!-- Misc --> - <i>com.yahoo.vespa:testutil</i> - <!-- Bundle plugin integration test --> - <i>com.yahoo.vespa.bundle-plugin:*</i> - </testUtilProjects> </enforceDependencies> </rules> <fail>true</fail> diff --git a/vespa-enforcer-extensions/pom.xml b/vespa-enforcer-extensions/pom.xml index 9d8e99156f3..d27dcc08a43 100644 --- a/vespa-enforcer-extensions/pom.xml +++ b/vespa-enforcer-extensions/pom.xml @@ -45,6 +45,11 @@ <artifactId>junit-jupiter</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>javax.inject</groupId> + <artifactId>javax.inject</artifactId> + <scope>provided</scope> + </dependency> </dependencies> <build> @@ -54,21 +59,18 @@ <artifactId>maven-compiler-plugin</artifactId> </plugin> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> + <!-- generate index of project components --> + <groupId>org.eclipse.sisu</groupId> + <artifactId>sisu-maven-plugin</artifactId> + <version>0.9.0.M2</version> <executions> <execution> - <phase>package</phase> <goals> - <goal>shade</goal> + <goal>main-index</goal> </goals> - <configuration> - <createDependencyReducedPom>false</createDependencyReducedPom> - </configuration> </execution> </executions> </plugin> - </plugins> </build> diff --git a/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/AllowedDependencies.java b/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/AllowedDependencies.java new file mode 100644 index 00000000000..656e2f52558 --- /dev/null +++ b/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/AllowedDependencies.java @@ -0,0 +1,305 @@ +package com.yahoo.vespa.maven.plugin.enforcer; + +import org.apache.maven.artifact.Artifact; +import org.apache.maven.enforcer.rule.api.AbstractEnforcerRule; +import org.apache.maven.enforcer.rule.api.EnforcerRule; +import org.apache.maven.enforcer.rule.api.EnforcerRuleException; +import org.apache.maven.enforcer.rule.api.EnforcerRuleHelper; +import org.apache.maven.execution.MavenSession; +import org.apache.maven.project.DefaultProjectBuildingRequest; +import org.apache.maven.project.MavenProject; +import org.apache.maven.shared.dependency.graph.DependencyGraphBuilder; +import org.apache.maven.shared.dependency.graph.DependencyGraphBuilderException; +import org.apache.maven.shared.dependency.graph.DependencyNode; +import org.codehaus.plexus.component.configurator.expression.ExpressionEvaluationException; +import org.codehaus.plexus.component.repository.exception.ComponentLookupException; + +import javax.inject.Inject; +import javax.inject.Named; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @author bjorncs + */ +@Named("allowedDependencies") +@SuppressWarnings("deprecation") +public class AllowedDependencies extends AbstractEnforcerRule implements EnforcerRule { + + private static final String WRITE_SPEC_PROP = "dependencyEnforcer.writeSpec"; + private static final String GUESS_VERSION = "dependencyEnforcer.guessProperty"; + + @Inject private MavenProject project; + @Inject private MavenSession session; + @Inject private DependencyGraphBuilder graphBuilder; + + // Injected parameters + public List<String> ignored; + public String rootProjectId; + public String specFile; + + @Override + public void execute(EnforcerRuleHelper helper) throws EnforcerRuleException { + try { + project = (MavenProject) helper.evaluate("${project}"); + session = (MavenSession) helper.evaluate("${session}"); + graphBuilder = helper.getComponent(DependencyGraphBuilder.class); + } catch (ExpressionEvaluationException | ComponentLookupException e) { + throw new RuntimeException(e); + } + execute(); + } + + public void execute() throws EnforcerRuleException { + var dependencies = getDependenciesOfAllProjects(); + getLog().info("Found %d unique dependencies ".formatted(dependencies.size())); + var specFile = Paths.get(project.getBasedir() + File.separator + this.specFile).normalize(); + var spec = loadDependencySpec(specFile); + var resolved = resolve(spec, dependencies); + if (System.getProperties().containsKey(WRITE_SPEC_PROP)) { + writeDependencySpec(specFile, resolved, System.getProperties().containsKey(GUESS_VERSION)); + getLog().info("Updated spec file '%s'".formatted(specFile.toString())); + } else { + warnOnDuplicateVersions(resolved); + validateDependencies(resolved, session.getRequest().getPom().toPath(), project.getArtifactId()); + } + getLog().info("The dependency enforcer completed successfully"); + } + + private static void validateDependencies(Resolved resolved, Path aggregatorPomRoot, String moduleName) + throws EnforcerRuleException { + if (!resolved.unmatchedRules().isEmpty() || !resolved.unmatchedDeps().isEmpty()) { + var errorMsg = new StringBuilder("The dependency enforcer failed:\n"); + if (!resolved.unmatchedRules().isEmpty()) { + errorMsg.append("Rules not matching any dependency:\n"); + resolved.unmatchedRules().forEach(r -> errorMsg.append(" - ").append(r.asString()).append('\n')); + } + if (!resolved.unmatchedDeps().isEmpty()) { + errorMsg.append("Dependencies not matching any rule:\n"); + resolved.unmatchedDeps().forEach(d -> errorMsg.append(" - ").append(d.asString(null)).append('\n')); + } + throw new EnforcerRuleException( + errorMsg.append("Maven dependency validation failed. ") + .append("If this change was intentional, update the dependency spec by running:\n") + .append("$ mvn validate -D").append(WRITE_SPEC_PROP).append(" -pl :").append(moduleName) + .append(" -f ").append(aggregatorPomRoot).append("\n").toString()); + } + } + + private Set<Dependency> getDependenciesOfAllProjects() throws EnforcerRuleException { + try { + Pattern depIgnorePattern = Pattern.compile( + ignored.stream() + .map(s -> s.replace(".", "\\.").replace("*", ".*").replace(":", "\\:").replace('?', '.')) + .collect(Collectors.joining(")|(", "^(", ")$"))); + List<MavenProject> projects = getAllProjects(session, rootProjectId); + Set<Dependency> dependencies = new HashSet<>(); + for (MavenProject project : projects) { + var req = new DefaultProjectBuildingRequest(session.getProjectBuildingRequest()); + req.setProject(project); + var root = graphBuilder.buildDependencyGraph(req, null); + addDependenciesRecursive(root, dependencies, depIgnorePattern); + } + return Set.copyOf(dependencies); + } catch (DependencyGraphBuilderException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + private static void addDependenciesRecursive(DependencyNode node, Set<Dependency> dependencies, Pattern ignored) { + if (node.getChildren() != null) { + for (DependencyNode dep : node.getChildren()) { + Artifact a = dep.getArtifact(); + Dependency dependency = Dependency.fromArtifact(a); + if (!ignored.matcher(dependency.asString(null)).matches()) { + dependencies.add(dependency); + } + addDependenciesRecursive(dep, dependencies, ignored); + } + } + } + + /** Only return the projects we'd like to enforce dependencies for: the root project, its modules, their modules, etc. */ + private static List<MavenProject> getAllProjects(MavenSession session, String rootProjectId) throws EnforcerRuleException { + if (rootProjectId == null) throw new EnforcerRuleException("Missing required <rootProjectId> in <enforceDependencies> in pom.xml"); + + List<MavenProject> allProjects = session.getAllProjects(); + if (allProjects.size() == 1) { + throw new EnforcerRuleException( + "Only a single Maven module detected. Enforcer must be executed from root of aggregator pom."); + } + MavenProject rootProject = allProjects + .stream() + .filter(project -> rootProjectId.equals(projectIdOf(project))) + .findAny() + .orElseThrow(() -> new EnforcerRuleException("Root project not found: " + rootProjectId)); + + Map<Path, MavenProject> projectsByBaseDir = allProjects + .stream() + .collect(Collectors.toMap(project -> project.getBasedir().toPath().normalize(), project -> project)); + + var projects = new ArrayList<MavenProject>(); + + var pendingProjects = new ArrayDeque<MavenProject>(); + pendingProjects.add(rootProject); + + while (!pendingProjects.isEmpty()) { + MavenProject project = pendingProjects.pop(); + projects.add(project); + + for (var module : project.getModules()) { + // Assumption: The module is a relative path to a project base directory. + Path moduleBaseDir = project.getBasedir().toPath().resolve(module).normalize(); + MavenProject moduleProject = projectsByBaseDir.get(moduleBaseDir); + if (moduleProject == null) + throw new EnforcerRuleException("Failed to find module '" + module + "' in project " + project.getBasedir()); + pendingProjects.add(moduleProject); + } + } + + projects.sort(Comparator.comparing(AllowedDependencies::projectIdOf)); + return projects; + } + + private List<Rule> loadDependencySpec(Path specFile) { + try (Stream<String> s = Files.lines(specFile)) { + return s.map(String::trim) + .filter(l -> !l.isEmpty() && !l.startsWith("#")) + .map(Rule::fromString) + .toList(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private Resolved resolve(List<Rule> spec, Set<Dependency> dependencies) { + var resolvedDeps = new HashSet<Dependency>(); + var resolveRules = new HashSet<Rule>(); + var unmatchedDeps = new HashSet<Dependency>(); + var unmatchedRules = new HashSet<Rule>(); + for (var rule : spec) { + var requiredDependency = rule.resolveToDependency(project.getProperties()); + if (dependencies.contains(requiredDependency)) { + resolvedDeps.add(requiredDependency); + resolveRules.add(rule); + } else { + unmatchedRules.add(rule); + } + } + for (var dependency : dependencies) { + if (!resolvedDeps.contains(dependency)) { + unmatchedDeps.add(dependency); + } + } + return new Resolved(resolvedDeps, resolveRules, unmatchedDeps, unmatchedRules); + } + + void writeDependencySpec(Path specFile, Resolved resolved, boolean guessVersion) { + var content = new TreeSet<String>(); + resolved.matchedRules().forEach(r -> content.add(r.asString())); + resolved.unmatchedDeps().forEach(d -> content.add(d.asString(guessVersion ? project.getProperties() : null))); + try (var out = Files.newBufferedWriter(specFile)) { + out.write("# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.\n\n"); + for (var line : content) { + out.write(line); out.write('\n'); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private void warnOnDuplicateVersions(Resolved resolved) { + Map<String, Set<String>> versionsForDependency = new TreeMap<>(); + Set<Dependency> allDeps = new HashSet<>(resolved.matchedDeps()); + allDeps.addAll(resolved.unmatchedDeps()); + for (Dependency d : allDeps) { + String id = "%s:%s".formatted(d.groupId(), d.artifactId()); + versionsForDependency.computeIfAbsent(id, __ -> new TreeSet<>()).add(d.version()); + } + versionsForDependency.forEach((dependency, versions) -> { + if (versions.size() > 1) { + getLog().warn("'%s' has multiple versions %s".formatted(dependency, versions)); + } + }); + } + + private static String projectIdOf(MavenProject project) { return "%s:%s".formatted(project.getGroupId(), project.getArtifactId()); } + + private record Rule(String groupId, String artifactId, String version, Optional<String> classifier){ + static final Pattern PROPERTY_PATTERN = Pattern.compile("\\$\\{(.+?)}"); + + static Rule fromString(String s) { + String[] splits = s.split(":"); + return splits.length == 3 + ? new Rule(splits[0], splits[1], splits[2], Optional.empty()) + : new Rule(splits[0], splits[1], splits[2], Optional.of(splits[3])); + } + + Dependency resolveToDependency(Properties props) { + // Replace expressions on form ${property} in 'version' field with value from properties + var matcher = PROPERTY_PATTERN.matcher(version); + var resolvedVersion = version; + while (matcher.find()) { + String property = matcher.group(1); + String value = props.getProperty(property); + if (value == null) throw new IllegalArgumentException("Missing property: " + property); + resolvedVersion = version.replace(matcher.group(), value); + } + return new Dependency(groupId, artifactId, resolvedVersion, classifier); + } + + String asString() { + var b = new StringBuilder(groupId).append(':').append(artifactId).append(':').append(version); + classifier.ifPresent(c -> b.append(':').append(c)); + return b.toString(); + } + } + + record Dependency(String groupId, String artifactId, String version, Optional<String> classifier) { + static Dependency fromArtifact(Artifact a) { + return new Dependency( + a.getGroupId(), a.getArtifactId(), a.getVersion(), Optional.ofNullable(a.getClassifier())); + } + + String asString(Properties props) { + String versionStr = version; + if (props != null) { + // Guess property name if properties are provided + var matchingProps = props.entrySet().stream() + .filter(e -> e.getValue().equals(version)) + .map(v -> "${%s}".formatted(v.getKey())) + .collect(Collectors.joining("|")); + if (!matchingProps.isEmpty()) versionStr = matchingProps; + } + var b = new StringBuilder(groupId).append(':').append(artifactId).append(':').append(versionStr); + classifier.ifPresent(c -> b.append(':').append(c)); + return b.toString(); + } + } + + record Resolved(Set<Dependency> matchedDeps, Set<Rule> matchedRules, + Set<Dependency> unmatchedDeps, Set<Rule> unmatchedRules) {} + + // Mark rule as not cachable + @Override public boolean isCacheable() { return false; } + @Override public boolean isResultValid(EnforcerRule r) { return false; } + @Override public String getCacheId() { return ""; } +} diff --git a/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesAllProjects.java b/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesAllProjects.java deleted file mode 100644 index 3db1019a2b1..00000000000 --- a/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesAllProjects.java +++ /dev/null @@ -1,319 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.maven.plugin.enforcer; - -import org.apache.maven.artifact.Artifact; -import org.apache.maven.enforcer.rule.api.EnforcerRule; -import org.apache.maven.enforcer.rule.api.EnforcerRuleException; -import org.apache.maven.enforcer.rule.api.EnforcerRuleHelper; -import org.apache.maven.execution.MavenSession; -import org.apache.maven.plugin.logging.Log; -import org.apache.maven.project.DefaultProjectBuildingRequest; -import org.apache.maven.project.MavenProject; -import org.apache.maven.shared.dependency.graph.DependencyGraphBuilder; -import org.apache.maven.shared.dependency.graph.DependencyGraphBuilderException; -import org.apache.maven.shared.dependency.graph.DependencyNode; -import org.codehaus.plexus.component.configurator.expression.ExpressionEvaluationException; -import org.codehaus.plexus.component.repository.exception.ComponentLookupException; - -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @author bjorncs - */ -@SuppressWarnings("deprecation") -public class EnforceDependenciesAllProjects implements EnforcerRule { - - private static final String WRITE_SPEC_PROP = "dependencyEnforcer.writeSpec"; - private static final String NON_TEST_HEADER = "#[non-test]"; - private static final String TEST_ONLY_HEADER = "#[test-only]"; - - private String rootProjectId; - private String specFile; - private List<String> ignored = List.of(); - private List<String> testUtilProjects = List.of(); - - @Override - public void execute(EnforcerRuleHelper helper) throws EnforcerRuleException { - Log log = helper.getLog(); - Dependencies deps = getDependenciesOfAllProjects(helper, ignored, testUtilProjects, rootProjectId); - log.info("Found %d unique dependencies (%d non-test, %d test only)".formatted( - deps.nonTest().size() + deps.testOnly().size(), deps.nonTest().size(), deps.testOnly().size())); - Path specFile = resolveSpecFile(helper, this.specFile); - if (System.getProperties().containsKey(WRITE_SPEC_PROP)) { - writeDependencySpec(specFile, deps); - log.info("Updated spec file '%s'".formatted(specFile.toString())); - } else { - warnOnDuplicateVersions(log, deps); - validateDependencies(deps, specFile, aggregatorPomRoot(helper), projectName(helper)); - } - log.info("The dependency enforcer completed successfully"); - } - - // Config injection for rule configuration. Method names must match config XML elements. - @SuppressWarnings("unused") public void setRootProjectId(String l) { this.rootProjectId = l; } - @SuppressWarnings("unused") public String getRootProjectId() { return rootProjectId; } - @SuppressWarnings("unused") public void setSpecFile(String f) { this.specFile = f; } - @SuppressWarnings("unused") public String getSpecFile() { return specFile; } - @SuppressWarnings("unused") public void setIgnored(List<String> l) { this.ignored = l; } - @SuppressWarnings("unused") public List<String> getIgnored() { return ignored; } - @SuppressWarnings("unused") public void setTestUtilProjects(List<String> l) { this.testUtilProjects = l; } - @SuppressWarnings("unused") public List<String> getTestUtilProjects() { return testUtilProjects; } - - record Dependency(String groupId, String artifactId, String version, Optional<String> classifier) - implements Comparable<Dependency> { - static Dependency fromArtifact(Artifact a) { - return new Dependency( - a.getGroupId(), a.getArtifactId(), a.getVersion(), Optional.ofNullable(a.getClassifier())); - } - - static Dependency fromString(String s) { - String[] splits = s.split(":"); - return splits.length == 3 - ? new Dependency(splits[0], splits[1], splits[2], Optional.empty()) - : new Dependency(splits[0], splits[1], splits[2], Optional.of(splits[3])); - } - - String asString() { - var b = new StringBuilder(groupId).append(':').append(artifactId).append(':').append(version); - classifier.ifPresent(c -> b.append(':').append(c)); - return b.toString(); - } - - static final Comparator<Dependency> COMPARATOR = Comparator.comparing(Dependency::groupId) - .thenComparing(Dependency::artifactId).thenComparing(Dependency::version) - .thenComparing(d -> d.classifier().orElse("")); - @Override public int compareTo(Dependency o) { return COMPARATOR.compare(this, o); } - } - - record Dependencies(SortedSet<Dependency> nonTest, SortedSet<Dependency> testOnly) {} - - static void validateDependencies(Dependencies dependencies, Path specFile, Path aggregatorPomRoot, - String moduleName) - throws EnforcerRuleException { - Dependencies allowedDependencies = loadDependencySpec(specFile); - if (!allowedDependencies.equals(dependencies)) { - StringBuilder errorMsg = new StringBuilder("The dependency enforcer failed:\n"); - generateDiff(errorMsg, "non-test", dependencies.nonTest(), allowedDependencies.nonTest()); - generateDiff(errorMsg, "test-only", dependencies.testOnly(), allowedDependencies.testOnly()); - throw new EnforcerRuleException( - errorMsg.append("Maven dependency validation failed. ") - .append("If this change was intentional, update the dependency spec by running:\n") - .append("$ mvn validate -D").append(WRITE_SPEC_PROP).append(" -pl :").append(moduleName) - .append(" -f ").append(aggregatorPomRoot).append("\n").toString()); - } - } - - static void generateDiff( - StringBuilder errorMsg, String label, SortedSet<Dependency> actual, SortedSet<Dependency> expected) { - SortedSet<Dependency> forbidden = new TreeSet<>(actual); - forbidden.removeAll(expected); - SortedSet<Dependency> removed = new TreeSet<>(expected); - removed.removeAll(actual); - if (!forbidden.isEmpty()) { - errorMsg.append("Forbidden ").append(label).append(" dependencies:\n"); - forbidden.forEach(d -> errorMsg.append(" - ").append(d.asString()).append('\n')); - } - if (!removed.isEmpty()) { - errorMsg.append("Removed ").append(label).append(" dependencies:\n"); - removed.forEach(d -> errorMsg.append(" - ").append(d.asString()).append('\n')); - } - } - - private static Dependencies getDependenciesOfAllProjects(EnforcerRuleHelper helper, List<String> ignored, - List<String> testUtilProjects, String rootProjectId) - throws EnforcerRuleException { - try { - Pattern depIgnorePattern = Pattern.compile( - ignored.stream() - .map(s -> s.replace(".", "\\.").replace("*", ".*").replace(":", "\\:").replace('?', '.')) - .collect(Collectors.joining(")|(", "^(", ")$"))); - Pattern projectIgnorePattern = Pattern.compile( - testUtilProjects.stream() - .map(s -> s.replace(".", "\\.").replace("*", ".*").replace(":", "\\:").replace('?', '.')) - .collect(Collectors.joining(")|(", "^(", ")$"))); - SortedSet<Dependency> nonTestDeps = new TreeSet<>(); - SortedSet<Dependency> testDeps = new TreeSet<>(); - MavenSession session = mavenSession(helper); - var graphBuilder = helper.getComponent(DependencyGraphBuilder.class); - List<MavenProject> projects = getAllProjects(session, rootProjectId); - for (MavenProject project : projects) { - var req = new DefaultProjectBuildingRequest(session.getProjectBuildingRequest()); - req.setProject(project); - DependencyNode root = graphBuilder.buildDependencyGraph(req, null); - String projectId = projectIdOf(project); - boolean overrideToTest = projectIgnorePattern.matcher(projectId).matches(); - if (overrideToTest) helper.getLog().info("Treating dependencies of '%s' as 'test'".formatted(projectId)); - addDependenciesRecursive(root, nonTestDeps, testDeps, depIgnorePattern, overrideToTest); - } - testDeps.removeAll(nonTestDeps); - return new Dependencies(nonTestDeps, testDeps); - } catch (DependencyGraphBuilderException | ComponentLookupException e) { - throw new RuntimeException(e.getMessage(), e); - } - } - - private static String projectIdOf(MavenProject project) { return "%s:%s".formatted(project.getGroupId(), project.getArtifactId()); } - - /** Only return the projects we'd like to enforce dependencies for: the root project, its modules, their modules, etc. */ - private static List<MavenProject> getAllProjects(MavenSession session, String rootProjectId) throws EnforcerRuleException { - if (rootProjectId == null) throw new EnforcerRuleException("Missing required <rootProjectId> in <enforceDependencies> in pom.xml"); - - List<MavenProject> allProjects = session.getAllProjects(); - if (allProjects.size() == 1) { - throw new EnforcerRuleException( - "Only a single Maven module detected. Enforcer must be executed from root of aggregator pom."); - } - MavenProject rootProject = allProjects - .stream() - .filter(project -> rootProjectId.equals(projectIdOf(project))) - .findAny() - .orElseThrow(() -> new EnforcerRuleException("Root project not found: " + rootProjectId)); - - Map<Path, MavenProject> projectsByBaseDir = allProjects - .stream() - .collect(Collectors.toMap(project -> project.getBasedir().toPath().normalize(), project -> project)); - - var projects = new ArrayList<MavenProject>(); - - var pendingProjects = new ArrayDeque<MavenProject>(); - pendingProjects.add(rootProject); - - while (!pendingProjects.isEmpty()) { - MavenProject project = pendingProjects.pop(); - projects.add(project); - - for (var module : project.getModules()) { - // Assumption: The module is a relative path to a project base directory. - Path moduleBaseDir = project.getBasedir().toPath().resolve(module).normalize(); - MavenProject moduleProject = projectsByBaseDir.get(moduleBaseDir); - if (moduleProject == null) - throw new EnforcerRuleException("Failed to find module '" + module + "' in project " + project.getBasedir()); - pendingProjects.add(moduleProject); - } - } - - projects.sort(Comparator.comparing(EnforceDependenciesAllProjects::projectIdOf)); - return projects; - } - - private static void addDependenciesRecursive( - DependencyNode node, Set<Dependency> nonTestDeps, Set<Dependency> testDeps, Pattern ignored, - boolean overrideToTest) { - if (node.getChildren() != null) { - for (DependencyNode dep : node.getChildren()) { - Artifact a = dep.getArtifact(); - Dependency dependency = Dependency.fromArtifact(a); - if (!ignored.matcher(dependency.asString()).matches()) { - if (a.getScope().equals("test") || overrideToTest) { - testDeps.add(dependency); - } else { - nonTestDeps.add(dependency); - } - } - addDependenciesRecursive(dep, nonTestDeps, testDeps, ignored, overrideToTest); - } - } - } - - private static void warnOnDuplicateVersions(Log log, Dependencies deps) { - Map<String, Set<String>> versionsForDependency = new TreeMap<>(); - Set<Dependency> allDeps = new TreeSet<>(deps.nonTest()); - allDeps.addAll(deps.testOnly()); - for (Dependency d : allDeps) { - String id = "%s:%s".formatted(d.groupId(), d.artifactId()); - versionsForDependency.computeIfAbsent(id, __ -> new TreeSet<>()).add(d.version()); - } - versionsForDependency.forEach((dependency, versions) -> { - if (versions.size() > 1) { - log.warn("'%s' has multiple versions %s".formatted(dependency, versions)); - } - }); - } - - private static Path resolveSpecFile(EnforcerRuleHelper helper, String specFile) { - return Paths.get(mavenProject(helper).getBasedir() + File.separator + specFile).normalize(); - } - - private static String projectName(EnforcerRuleHelper helper) { return mavenProject(helper).getArtifactId(); } - - private static Path aggregatorPomRoot(EnforcerRuleHelper helper) { - return mavenSession(helper).getRequest().getPom().toPath(); - } - - private static MavenProject mavenProject(EnforcerRuleHelper helper) { - try { - return (MavenProject) helper.evaluate("${project}"); - } catch (ExpressionEvaluationException e) { - throw new RuntimeException(e.getMessage(), e); - } - } - - private static MavenSession mavenSession(EnforcerRuleHelper helper) { - try { - return (MavenSession) helper.evaluate("${session}"); - } catch (ExpressionEvaluationException e) { - throw new RuntimeException(e.getMessage(), e); - } - } - - static void writeDependencySpec(Path specFile, Dependencies dependencies) { - try (var out = Files.newBufferedWriter(specFile)) { - out.write("# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.\n\n"); - out.write(NON_TEST_HEADER); out.write('\n'); - out.write("# Contains dependencies that are not used exclusively in 'test' scope\n"); - for (Dependency d : dependencies.nonTest()) { - out.write(d.asString()); out.write('\n'); - } - out.write("\n"); out.write(TEST_ONLY_HEADER); out.write('\n'); - out.write("# Contains dependencies that are used exclusively in 'test' scope\n"); - for (Dependency d : dependencies.testOnly()) { - out.write(d.asString()); out.write('\n'); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static Dependencies loadDependencySpec(Path specFile) { - try { - List<String> lines; - try (Stream<String> s = Files.lines(specFile)) { - lines = s.map(String::trim).filter(l -> !l.isEmpty()).toList(); - } - SortedSet<Dependency> nonTest = parseDependencies(lines.stream().takeWhile(l -> !l.equals(TEST_ONLY_HEADER))); - SortedSet<Dependency> testOnly = parseDependencies(lines.stream().dropWhile(l -> !l.equals(TEST_ONLY_HEADER))); - return new Dependencies(nonTest, testOnly); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static SortedSet<Dependency> parseDependencies(Stream<String> lines) { - return lines.filter(l -> !l.startsWith("#")).map(Dependency::fromString) - .collect(Collectors.toCollection(TreeSet::new)); - } - - // Mark rule as not cachable - @Override public boolean isCacheable() { return false; } - @Override public boolean isResultValid(EnforcerRule r) { return false; } - @Override public String getCacheId() { return ""; } - -} diff --git a/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesAllProjectsTest.java b/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesAllProjectsTest.java deleted file mode 100644 index 59062cbd61c..00000000000 --- a/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesAllProjectsTest.java +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.maven.plugin.enforcer; - -import com.yahoo.vespa.maven.plugin.enforcer.EnforceDependenciesAllProjects.Dependencies; -import com.yahoo.vespa.maven.plugin.enforcer.EnforceDependenciesAllProjects.Dependency; -import org.apache.maven.enforcer.rule.api.EnforcerRuleException; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; - -import static com.yahoo.vespa.maven.plugin.enforcer.EnforceDependenciesAllProjects.validateDependencies; -import static com.yahoo.vespa.maven.plugin.enforcer.EnforceDependenciesAllProjects.writeDependencySpec; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author bjorncs - */ -class EnforceDependenciesAllProjectsTest { - - private static final Path POM_FILE = Paths.get("/vespa-src/pom.xml"); - - @Test - void succeeds_dependencies_matches_spec() { - SortedSet<Dependency> nonTest = new TreeSet<>(Set.of( - Dependency.fromString("com.example:foo:1.2.3"), - Dependency.fromString("com.example:bar:2.3.4"))); - SortedSet<Dependency> testOnly = new TreeSet<>(Set.of( - Dependency.fromString("com.example:testfoo:1.2.3"), - Dependency.fromString("com.example:testbar:2.3.4"))); - Path specFile = Paths.get("src/test/resources/allowed-dependencies.txt"); - Dependencies deps = new Dependencies(nonTest, testOnly); - assertDoesNotThrow(() -> validateDependencies(deps, specFile, POM_FILE, "my-dep-enforcer")); - } - - @Test - void fails_on_forbidden_dependency() { - SortedSet<Dependency> nonTest = new TreeSet<>(Set.of( - Dependency.fromString("com.example:foo:1.2.3"), - Dependency.fromString("com.example:bar:2.3.4"), - Dependency.fromString("com.example:foobar:3.4.5"))); - SortedSet<Dependency> testOnly = new TreeSet<>(Set.of( - Dependency.fromString("com.example:testfoo:1.2.3"), - Dependency.fromString("com.example:testbar:2.3.4"))); - Path specFile = Paths.get("src/test/resources/allowed-dependencies.txt"); - Dependencies deps = new Dependencies(nonTest, testOnly); - var exception = assertThrows(EnforcerRuleException.class, - () -> validateDependencies(deps, specFile, POM_FILE, "my-dep-enforcer")); - String expectedErrorMessage = - """ - The dependency enforcer failed: - Forbidden non-test dependencies: - - com.example:foobar:3.4.5 - Maven dependency validation failed. If this change was intentional, update the dependency spec by running: - $ mvn validate -DdependencyEnforcer.writeSpec -pl :my-dep-enforcer -f /vespa-src/pom.xml - """; - assertEquals(expectedErrorMessage, exception.getMessage()); - } - - @Test - void fails_on_missing_dependency() { - SortedSet<Dependency> nonTest = new TreeSet<>(Set.of( - Dependency.fromString("com.example:bar:2.3.4"))); - SortedSet<Dependency> testOnly = new TreeSet<>(Set.of( - Dependency.fromString("com.example:testfoo:1.2.3"))); - Path specFile = Paths.get("src/test/resources/allowed-dependencies.txt"); - Dependencies deps = new Dependencies(nonTest, testOnly); - var exception = assertThrows(EnforcerRuleException.class, - () -> validateDependencies(deps, specFile, POM_FILE, "my-dep-enforcer")); - String expectedErrorMessage = - """ - The dependency enforcer failed: - Removed non-test dependencies: - - com.example:foo:1.2.3 - Removed test-only dependencies: - - com.example:testbar:2.3.4 - Maven dependency validation failed. If this change was intentional, update the dependency spec by running: - $ mvn validate -DdependencyEnforcer.writeSpec -pl :my-dep-enforcer -f /vespa-src/pom.xml - """; - assertEquals(expectedErrorMessage, exception.getMessage()); - } - - @Test - void writes_valid_spec_file(@TempDir Path tempDir) throws IOException { - SortedSet<Dependency> nonTest = new TreeSet<>(Set.of( - Dependency.fromString("com.example:foo:1.2.3"), - Dependency.fromString("com.example:bar:2.3.4"))); - SortedSet<Dependency> testOnly = new TreeSet<>(Set.of( - Dependency.fromString("com.example:testfoo:1.2.3"), - Dependency.fromString("com.example:testbar:2.3.4"))); - Dependencies deps = new Dependencies(nonTest, testOnly); - Path outputFile = tempDir.resolve("allowed-dependencies.txt"); - writeDependencySpec(outputFile, deps); - assertEquals( - Files.readString(Paths.get("src/test/resources/allowed-dependencies.txt")), - Files.readString(outputFile)); - - } - -}
\ No newline at end of file diff --git a/vespa-enforcer-extensions/src/test/resources/allowed-dependencies.txt b/vespa-enforcer-extensions/src/test/resources/allowed-dependencies.txt deleted file mode 100644 index 2ef0f9e0c0c..00000000000 --- a/vespa-enforcer-extensions/src/test/resources/allowed-dependencies.txt +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#[non-test] -# Contains dependencies that are not used exclusively in 'test' scope -com.example:bar:2.3.4 -com.example:foo:1.2.3 - -#[test-only] -# Contains dependencies that are used exclusively in 'test' scope -com.example:testbar:2.3.4 -com.example:testfoo:1.2.3 diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index c1d7e17b457..56dcd9abdf6 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -96,6 +96,7 @@ vespa_define_module( src/tests/fileheader src/tests/floatingpointtype src/tests/fuzzy + src/tests/fuzzy/table_dfa src/tests/gencnt src/tests/growablebytebuffer src/tests/guard diff --git a/vespalib/src/tests/fuzzy/levenshtein_dfa_test.cpp b/vespalib/src/tests/fuzzy/levenshtein_dfa_test.cpp index c235cb99509..919ba328085 100644 --- a/vespalib/src/tests/fuzzy/levenshtein_dfa_test.cpp +++ b/vespalib/src/tests/fuzzy/levenshtein_dfa_test.cpp @@ -11,6 +11,7 @@ #include <string> #include <string_view> #include <gtest/gtest.h> +#include <gmock/gmock.h> using namespace ::testing; using namespace vespalib::fuzzy; @@ -82,7 +83,8 @@ INSTANTIATE_TEST_SUITE_P(AllCasingAndDfaTypes, Combine(Values(LevenshteinDfa::Casing::Uncased, LevenshteinDfa::Casing::Cased), Values(LevenshteinDfa::DfaType::Explicit, - LevenshteinDfa::DfaType::Implicit)), + LevenshteinDfa::DfaType::Implicit, + LevenshteinDfa::DfaType::Table)), LevenshteinDfaTest::stringify_params); // Same as existing non-DFA Levenshtein tests, but with some added instantiations @@ -122,8 +124,10 @@ TEST_P(LevenshteinDfaTest, distance_is_in_utf32_code_point_space) { EXPECT_EQ(calculate(u8"カラオケ", u8"カラoke", 2), std::nullopt); } -void test_dfa_successor(const LevenshteinDfa& dfa, std::string_view source, std::string_view expected_successor) { - std::string successor; +void test_dfa_successor(const LevenshteinDfa& dfa, std::string_view source, + std::string_view expected_successor, std::string_view successor_prefix) +{ + std::string successor(successor_prefix); auto m = dfa.match(source, successor); if (m.matches()) { FAIL() << "Expected '" << source << "' to emit a successor, but it " @@ -131,15 +135,21 @@ void test_dfa_successor(const LevenshteinDfa& dfa, std::string_view source, std: << " edits (of max " << static_cast<uint32_t>(m.max_edits()) << " edits)"; } EXPECT_EQ(successor, expected_successor); - EXPECT_TRUE(dfa.match(successor).matches()); + // Must skip any caller-provided successor prefix before checking if it matches the target + auto successor_suffix = successor.substr(successor_prefix.size()); + EXPECT_TRUE(dfa.match(successor_suffix).matches()); // Make sure the UTF-32 successor output is codepoint-wise identical to the UTF-8 successor - std::vector<uint32_t> u32successor; + std::vector<uint32_t> u32successor(utf8_string_to_utf32(successor_prefix)); m = dfa.match(source, u32successor); EXPECT_FALSE(m.matches()); expect_utf32_string_code_point_equal_to_utf8(u32successor, successor); } +void test_dfa_successor(const LevenshteinDfa& dfa, std::string_view source, std::string_view expected_successor) { + test_dfa_successor(dfa, source, expected_successor, {}); +} + TEST_P(LevenshteinDfaTest, can_generate_successors_to_mismatching_source_strings) { auto dfa = LevenshteinDfa::build("food", 1, casing(), dfa_type()); @@ -201,6 +211,28 @@ TEST_P(LevenshteinDfaTest, successor_is_well_defined_for_empty_target) { test_dfa_successor(dfa, "vespa", "w"); } +TEST_P(LevenshteinDfaTest, caller_provided_successor_prefix_is_preserved_on_mismatch) { + auto dfa = LevenshteinDfa::build("food", 1, casing(), dfa_type()); + + // Same inputs as existing successor tests, but with a preserved prefix in the generated successor + test_dfa_successor(dfa, "", "yolo\x01""food", "yolo"); + test_dfa_successor(dfa, "faa", "xyzfaod", "xyz"); + test_dfa_successor(dfa, "fooooo", "ABCfoop", "ABC"); + test_dfa_successor(dfa, "ooof", "ABCpfood", "ABC"); + test_dfa_successor(dfa, "gp", "yolohfood", "yolo"); + + dfa = LevenshteinDfa::build("", 1, casing(), dfa_type()); + test_dfa_successor(dfa, "aa", "foob", "foo"); +} + +TEST_P(LevenshteinDfaTest, caller_provided_successor_prefix_is_preserved_on_match) { + auto dfa = LevenshteinDfa::build("food", 1, casing(), dfa_type()); + std::string successor = "bar"; + auto m = dfa.match("mood", successor); + EXPECT_TRUE(m.matches()); + EXPECT_THAT(successor, StartsWith("bar")); +} + // We should normally be able to rely on higher-level components to ensure we // only receive valid UTF-8, but make sure we don't choke on it if we do get it. TEST_P(LevenshteinDfaTest, malformed_utf8_is_replaced_with_placeholder_char) { @@ -233,7 +265,8 @@ struct LevenshteinDfaCasingTest : TestWithParam<LevenshteinDfa::DfaType> { INSTANTIATE_TEST_SUITE_P(AllDfaTypes, LevenshteinDfaCasingTest, Values(LevenshteinDfa::DfaType::Explicit, - LevenshteinDfa::DfaType::Implicit), + LevenshteinDfa::DfaType::Implicit, + LevenshteinDfa::DfaType::Table), PrintToStringParamName()); TEST_P(LevenshteinDfaCasingTest, uncased_edge_cases_have_correct_edit_distance) { @@ -315,7 +348,8 @@ INSTANTIATE_TEST_SUITE_P(SupportedMaxEdits, Combine(Values(LevenshteinDfa::Casing::Uncased, LevenshteinDfa::Casing::Cased), Values(LevenshteinDfa::DfaType::Explicit, - LevenshteinDfa::DfaType::Implicit), + LevenshteinDfa::DfaType::Implicit, + LevenshteinDfa::DfaType::Table), Values(1, 2)), LevenshteinDfaSuccessorTest::stringify_params); @@ -342,6 +376,7 @@ TEST_P(LevenshteinDfaSuccessorTest, exhaustive_successor_test) { std::string skip_to, successor; for (uint32_t j = 0; j < 256; ++j) { const auto source = bits_to_str(static_cast<uint8_t>(j)); + successor.clear(); auto maybe_match = target_dfa.match(source, successor); if (maybe_match.matches() && !skip_to.empty()) { ASSERT_GE(source, skip_to); @@ -597,6 +632,7 @@ TEST_P(LevenshteinBenchmarkTest, benchmark_skipping_dictionary_scan) { auto end = dict.cend(); std::string successor; while (iter != end) { + successor.clear(); auto maybe_match = dfa.match(*iter, successor); if (maybe_match.matches()) { ++iter; diff --git a/vespalib/src/tests/fuzzy/table_dfa/CMakeLists.txt b/vespalib/src/tests/fuzzy/table_dfa/CMakeLists.txt new file mode 100644 index 00000000000..1017ac99564 --- /dev/null +++ b/vespalib/src/tests/fuzzy/table_dfa/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_fuzzy_table_dfa_test_app TEST + SOURCES + table_dfa_test.cpp + DEPENDS + vespalib + GTest::GTest + ) +vespa_add_test(NAME vespalib_fuzzy_table_dfa_test_app COMMAND vespalib_fuzzy_table_dfa_test_app) diff --git a/vespalib/src/tests/fuzzy/table_dfa/table_dfa_test.cpp b/vespalib/src/tests/fuzzy/table_dfa/table_dfa_test.cpp new file mode 100644 index 00000000000..acb68a56de7 --- /dev/null +++ b/vespalib/src/tests/fuzzy/table_dfa/table_dfa_test.cpp @@ -0,0 +1,395 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/fuzzy/table_dfa.hpp> +#include <vespa/vespalib/gtest/gtest.h> +#include <set> + +using namespace ::testing; +using namespace vespalib::fuzzy; + +// test/experiment with low-level concepts underlying the construction +// of the tables used in the table-driven dfa implementation. + +TEST(TableDfaTest, position) { + Position pos1 = Position::start(); + EXPECT_EQ(pos1.index, 0); + EXPECT_EQ(pos1.edits, 0); + Position pos2(2, 3); + EXPECT_EQ(pos2.index, 2); + EXPECT_EQ(pos2.edits, 3); +} + +TEST(TableDfaTest, position_equality) { + Position pos1(0, 0); + Position pos2(0, 1); + Position pos3(1, 0); + EXPECT_TRUE(pos1 == pos1); + EXPECT_FALSE(pos1 == pos2); + EXPECT_FALSE(pos1 == pos2); +} + +TEST(TableDfaTest, position_sort_order) { + std::vector<Position> list; + list.emplace_back(0,1); + list.emplace_back(0,0); + list.emplace_back(1,0); + list.emplace_back(1,1); + std::sort(list.begin(), list.end()); + EXPECT_EQ(list[0].index, 0); + EXPECT_EQ(list[0].edits, 0); + EXPECT_EQ(list[1].index, 1); + EXPECT_EQ(list[1].edits, 0); + EXPECT_EQ(list[2].index, 0); + EXPECT_EQ(list[2].edits, 1); + EXPECT_EQ(list[3].index, 1); + EXPECT_EQ(list[3].edits, 1); +} + +TEST(TableDfaTest, position_subsumption) { + Position pos1(0, 0); + Position pos2(0, 1); + Position pos3(0, 2); + + Position pos4(1, 0); + Position pos5(1, 1); + Position pos6(1, 2); + + Position pos7(2, 0); + Position pos8(2, 1); + Position pos9(2, 2); + + EXPECT_FALSE(pos1.subsumes(pos1)); + EXPECT_TRUE(pos1.subsumes(pos2)); + EXPECT_TRUE(pos1.subsumes(pos3)); + EXPECT_FALSE(pos1.subsumes(pos4)); + EXPECT_TRUE(pos1.subsumes(pos5)); + EXPECT_TRUE(pos1.subsumes(pos6)); + EXPECT_FALSE(pos1.subsumes(pos7)); + EXPECT_FALSE(pos1.subsumes(pos8)); + EXPECT_TRUE(pos1.subsumes(pos9)); + + EXPECT_FALSE(pos5.subsumes(pos1)); + EXPECT_FALSE(pos5.subsumes(pos2)); + EXPECT_TRUE(pos5.subsumes(pos3)); + EXPECT_FALSE(pos5.subsumes(pos4)); + EXPECT_FALSE(pos5.subsumes(pos5)); + EXPECT_TRUE(pos5.subsumes(pos6)); + EXPECT_FALSE(pos5.subsumes(pos7)); + EXPECT_FALSE(pos5.subsumes(pos8)); + EXPECT_TRUE(pos5.subsumes(pos9)); +} + +TEST(TableDfaTest, position_materialization) { + EXPECT_EQ(Position(1,1).materialize(0).index, 0); + EXPECT_EQ(Position(1,1).materialize(1).index, 1); + EXPECT_EQ(Position(1,1).materialize(2).index, 2); + EXPECT_EQ(Position(1,1).materialize(0).edits, 2); + EXPECT_EQ(Position(1,1).materialize(1).edits, 1); + EXPECT_EQ(Position(1,1).materialize(2).edits, 2); +} + +TEST(TableDfaTest, position_to_string) { + Position pos1(0, 0); + Position pos2(1, 2); + Position pos3(2, 3); + EXPECT_EQ(pos1.to_string(), fmt("0#0")); + EXPECT_EQ(pos2.to_string(), fmt("1#2")); + EXPECT_EQ(pos3.to_string(), fmt("2#3")); +} + +TEST(TableDfaTest, state_creation_reorder) { + EXPECT_EQ(State::create<5>({{0,1},{2,0}}).to_string(), fmt("{2#0,0#1}")); + EXPECT_EQ(State::create<5>({{2,0},{0,0}}).to_string(), fmt("{0#0,2#0}")); +} + +TEST(TableDfaTest, state_creation_duplicate_removal) { + EXPECT_EQ(State::create<5>({{0,0},{0,0},{2,1},{2,1}}).to_string(), fmt("{0#0,2#1}")); +} + +TEST(TableDfaTest, state_creation_edit_cutoff) { + EXPECT_EQ(State::create<2>({{0,0},{5,2},{10,3}}).to_string(), fmt("{0#0,5#2}")); +} + +TEST(TableDfaTest, state_creation_subsumption_collapsing) { + EXPECT_EQ(State::create<2>({{0,0},{1,1}}).to_string(), fmt("{0#0}")); + EXPECT_EQ(State::create<2>({{0,1},{1,0}}).to_string(), fmt("{1#0}")); + EXPECT_EQ(State::create<2>({{0,0},{2,2}}).to_string(), fmt("{0#0}")); + EXPECT_EQ(State::create<2>({{0,2},{2,0}}).to_string(), fmt("{2#0}")); +} + +TEST(TableDfaTest, state_normalization) { + auto state1 = State::create<2>({{2,1},{3,1}}); + auto state2 = State::create<2>({{5,0},{3,1}}); + EXPECT_EQ(state1.to_string(), fmt("{2#1,3#1}")); + EXPECT_EQ(state2.to_string(), fmt("{5#0,3#1}")); + EXPECT_EQ(state1.normalize(), 2); + EXPECT_EQ(state2.normalize(), 3); + EXPECT_EQ(state1.to_string(), fmt("{0#1,1#1}")); + EXPECT_EQ(state2.to_string(), fmt("{2#0,0#1}")); +} + +TEST(TableDfaTest, state_repo) { + StateRepo repo; + EXPECT_EQ(repo.state_to_idx(State::failed()), 0); + EXPECT_EQ(repo.state_to_idx(State::start()), 1); + EXPECT_EQ(repo.state_to_idx(State::create<2>({{0,0},{1,0}})), 2); + EXPECT_EQ(repo.state_to_idx(State::create<2>({{0,0},{2,1}})), 3); + EXPECT_EQ(repo.state_to_idx(State::create<2>({{0,0},{1,0}})), 2); + EXPECT_EQ(repo.state_to_idx(State::create<2>({{0,0},{2,1}})), 3); + EXPECT_EQ(repo.size(), 4); + EXPECT_EQ(repo.idx_to_state(0).to_string(), fmt("{}")); + EXPECT_EQ(repo.idx_to_state(1).to_string(), fmt("{0#0}")); + EXPECT_EQ(repo.idx_to_state(2).to_string(), fmt("{0#0,1#0}")); + EXPECT_EQ(repo.idx_to_state(3).to_string(), fmt("{0#0,2#1}")); +} + +TEST(TableDfaTest, expand_bits) { + auto yes = expand_bits<2>(0x1f); + auto no = expand_bits<2>(0x00); + auto odd = expand_bits<2>(0x0a); + auto even = expand_bits<2>(0x15); + ASSERT_EQ(yes.size(), 5); + ASSERT_EQ(no.size(), 5); + ASSERT_EQ(odd.size(), 5); + ASSERT_EQ(even.size(), 5); + for (size_t i = 0; i < 5; ++i) { + EXPECT_TRUE(yes[i]); + EXPECT_FALSE(no[i]); + EXPECT_EQ(odd[i], bool(i % 2 == 1)); + EXPECT_EQ(even[i], bool(i % 2 == 0)); + } +} + +TEST(TableDfaTest, format_bits) { + EXPECT_EQ(format_vector(expand_bits<1>(0)), fmt("[0,0,0]")); + EXPECT_EQ(format_vector(expand_bits<1>(7)), fmt("[1,1,1]")); + EXPECT_EQ(format_vector(expand_bits<1>(5)), fmt("[1,0,1]")); + EXPECT_EQ(format_vector(expand_bits<1>(2)), fmt("[0,1,0]")); + EXPECT_EQ(format_vector(expand_bits<2>(31)), fmt("[1,1,1,1,1]")); + EXPECT_EQ(format_vector(expand_bits<2>(21)), fmt("[1,0,1,0,1]")); + EXPECT_EQ(format_vector(expand_bits<2>(31), true), fmt("11111")); + EXPECT_EQ(format_vector(expand_bits<2>(21), true), fmt("10101")); +} + +template <uint8_t N> +void list_states() { + auto repo = make_state_repo<N>(); + EXPECT_EQ(num_states<N>(), repo.size()); + fprintf(stderr, "max_edits: %u, number of states: %zu\n", N, repo.size()); + for (uint32_t i = 0; i < repo.size(); ++i) { + fprintf(stderr, " state %u: %s\n", i, repo.idx_to_state(i).to_string().c_str()); + } +} + +TEST(TableDfaTest, list_states_for_max_edits_1) { list_states<1>(); } +TEST(TableDfaTest, list_states_for_max_edits_2) { list_states<2>(); } + +template <uint8_t N> +void list_edits() { + auto repo = make_state_repo<N>(); + fprintf(stderr, + "per state, listing the minimal number of edits needed\n" + "to reach offsets at and beyond its minimal boundary\n"); + for (uint32_t i = 0; i < repo.size(); ++i) { + const State &state = repo.idx_to_state(i); + fprintf(stderr, "%-23s : %s\n", state.to_string().c_str(), + format_vector(state.make_edit_vector<N>()).c_str()); + } +} + +TEST(TableDfaTest, list_edits_at_input_end_for_max_edits_1) { list_edits<1>(); } +TEST(TableDfaTest, list_edits_at_input_end_for_max_edits_2) { list_edits<2>(); } + +template <uint8_t N> +void list_transitions() { + auto repo = make_state_repo<N>(); + for (uint32_t idx = 0; idx < repo.size(); ++idx) { + const State &state = repo.idx_to_state(idx); + for (uint32_t i = 0; i < num_transitions<N>(); ++i) { + auto bits = expand_bits<N>(i); + State new_state = state.next<N>(bits); + uint32_t step = new_state.normalize(); + uint32_t new_idx = repo.state_to_idx(new_state); + ASSERT_LT(new_idx, repo.size()); + fprintf(stderr, "%u:%s,i --%s--> %u:%s,%s\n", idx, state.to_string().c_str(), + format_vector(bits).c_str(), new_idx, new_state.to_string().c_str(), + (step == 0) ? "i" : fmt("i+%u", step).c_str()); + } + } +} + +TEST(TableDfaTest, list_transitions_for_max_edits_1) { list_transitions<1>(); } + +// Simulate all possible ways we can approach the end of the word we +// are matching. Verify that no transition taken can produce a state +// with a minimal boundary that exceeds the boundary of the word +// itself. Verifying this will enable us to not care about word size +// while simulating the dfa. +template <uint8_t N> +void verify_word_end_boundary() { + auto repo = make_state_repo<N>(); + using StateSet = std::set<uint32_t>; + std::vector<StateSet> active(window_size<N>() + 1); + for (size_t i = 1; i < repo.size(); ++i) { + active[0].insert(i); + } + EXPECT_EQ(active.size(), window_size<N>() + 1); + EXPECT_EQ(active[0].size(), repo.size() - 1); + fprintf(stderr, "verifying word end for max edits %u\n", N); + uint32_t edge_shape = 0; + for (uint32_t active_idx = 0; active_idx < active.size(); ++active_idx) { + fprintf(stderr, " edge shape: %s, max step: %zu, active_states: %zu\n", + format_vector(expand_bits<N>(edge_shape)).c_str(), active.size() - active_idx - 1, active[active_idx].size()); + for (uint32_t idx: active[active_idx]) { + const State &state = repo.idx_to_state(idx); + for (uint32_t i = 0; i < num_transitions<N>(); ++i) { + if ((i & edge_shape) == 0) { + State new_state = state.next<N>(expand_bits<N>(i)); + uint32_t step = new_state.normalize(); + uint32_t new_idx = repo.state_to_idx(new_state); + ASSERT_LT(new_idx, repo.size()); + if (new_idx != 0) { + ASSERT_GT(active.size(), active_idx + step); + active[active_idx + step].insert(new_idx); + } + } + } + } + edge_shape = (edge_shape << 1) + 1; + } + EXPECT_EQ(edge_shape, (1 << (window_size<N>() + 1)) - 1); + while (!active.back().empty()) { + fprintf(stderr, " residue states after word end: %zu\n", active.back().size()); + StateSet residue; + for (uint32_t idx: active.back()) { + const State &state = repo.idx_to_state(idx); + State new_state = state.next<N>(expand_bits<N>(0)); + uint32_t step = new_state.normalize(); + uint32_t new_idx = repo.state_to_idx(new_state); + ASSERT_LT(new_idx, repo.size()); + ASSERT_EQ(step, 0); + if (new_idx != 0) { + residue.insert(new_idx); + } + } + active.back() = std::move(residue); + } +} + +TEST(TableDfaTest, minimal_boundary_will_never_exceed_word_end_with_max_edits_1) { + verify_word_end_boundary<1>(); +} + +TEST(TableDfaTest, minimal_boundary_will_never_exceed_word_end_with_max_edits_2) { + verify_word_end_boundary<2>(); +} + +template <uint8_t N> +void verify_inline_tfa() { + auto tfa = make_tfa<N>(); + fprintf(stderr, "verifying TFA for N = %u (byte size: %zu)\n", N, sizeof(*tfa)); + ASSERT_EQ(tfa->table.size(), num_states<N>()); + ASSERT_EQ(tfa->edits.size(), num_states<N>()); + for (size_t state = 0; state < num_states<N>(); ++state) { + ASSERT_EQ(tfa->table[state].size(), num_transitions<N>()); + for (size_t transition = 0; transition < num_transitions<N>(); ++transition) { + EXPECT_EQ(tfa->table[state][transition].step, InlineTfa<N>::table[state][transition].step); + EXPECT_EQ(tfa->table[state][transition].state, InlineTfa<N>::table[state][transition].state); + } + ASSERT_EQ(tfa->edits[state].size(), window_size<N>()); + for (size_t offset = 0; offset < window_size<N>(); ++offset) { + EXPECT_EQ(tfa->edits[state][offset], InlineTfa<N>::edits[state][offset]); + } + } +} + +TEST(TableDfaTest, verify_inline_tfa_with_max_edits_1) { + verify_inline_tfa<1>(); +} + +TEST(TableDfaTest, verify_inline_tfa_with_max_edits_2) { + verify_inline_tfa<2>(); +} + +template <uint8_t N> +void dump_tfa_as_code() { + auto tfa = make_tfa<N>(); + fprintf(stderr, "// start of auto-generated code for N = %u\n", N); + fprintf(stderr, "template <> struct InlineTfa<%u> {\n", N); + fprintf(stderr, " static constexpr Transition table[%zu][%zu] = {\n", num_states<N>(), num_transitions<N>()); + for (size_t state = 0; state < num_states<N>(); ++state) { + fprintf(stderr, " {"); + for (size_t transition = 0; transition < num_transitions<N>(); ++transition) { + if (transition > 0) { + fprintf(stderr, ","); + } + fprintf(stderr, "{%u,%u}", tfa->table[state][transition].step, tfa->table[state][transition].state); + } + fprintf(stderr, "}%s\n", ((state + 1) < num_states<N>()) ? "," : ""); + } + fprintf(stderr, " };\n"); + fprintf(stderr, " static constexpr uint8_t edits[%zu][%zu] = {\n", num_states<N>(), window_size<N>()); + for (size_t state = 0; state < num_states<N>(); ++state) { + fprintf(stderr, " {"); + for (size_t offset = 0; offset < window_size<N>(); ++offset) { + if (offset > 0) { + fprintf(stderr, ","); + } + fprintf(stderr, "%u", tfa->edits[state][offset]); + } + fprintf(stderr, "}%s\n", ((state + 1) < num_states<N>()) ? "," : ""); + } + fprintf(stderr, " };\n"); + fprintf(stderr, "};\n"); + fprintf(stderr, "// end of auto-generated code for N = %u\n", N); +} + +TEST(TableDfaTest, dump_tfa_with_max_edits_1_as_code) { + dump_tfa_as_code<1>(); +} + +TEST(TableDfaTest, dump_tfa_with_max_edits_2_as_code) { + dump_tfa_as_code<2>(); +} + +template <uint8_t N> +void dump_tfa_graph() { + auto repo = make_state_repo<N>(); + fprintf(stderr, "digraph tfa {\n"); + for (uint32_t idx = 0; idx < repo.size(); ++idx) { + fprintf(stderr, " %u [label=\"%s\"];\n", idx, + repo.idx_to_state(idx).to_string().c_str()); + } + // omit transitions from the failure state to itself + for (uint32_t idx = 1; idx < repo.size(); ++idx) { + const State &state = repo.idx_to_state(idx); + for (uint32_t i = 0; i < num_transitions<N>(); ++i) { + auto bits = expand_bits<N>(i); + State new_state = state.next<N>(bits); + uint32_t step = new_state.normalize(); + uint32_t new_idx = repo.state_to_idx(new_state); + ASSERT_LT(new_idx, repo.size()); + if (bits[0] && idx == new_idx && step == 1) { + // omit simple transitions to yourself + } else { + fprintf(stderr, " %u -> %u [label=\"%s,%u\"];\n", idx, new_idx, + format_vector(bits, true).c_str(), step); + } + } + } + fprintf(stderr, "}\n"); +} + +TEST(TableDfaTest, graphviz_for_tfa_with_max_edits_1) { + dump_tfa_graph<1>(); +} + +TEST(TableDfaTest, graphviz_for_food_with_max_edits_1) { + auto dfa = LevenshteinDfa::build("food", 1, LevenshteinDfa::Casing::Cased, LevenshteinDfa::DfaType::Table); + std::ostringstream out; + dfa.dump_as_graphviz(out); + fprintf(stderr, "memory usage: %zu\n", dfa.memory_usage()); + fprintf(stderr, "%s", out.str().c_str()); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/memorydatastore/memorydatastore.cpp b/vespalib/src/tests/memorydatastore/memorydatastore.cpp index 1d49b0af91b..7eab32601de 100644 --- a/vespalib/src/tests/memorydatastore/memorydatastore.cpp +++ b/vespalib/src/tests/memorydatastore/memorydatastore.cpp @@ -6,19 +6,9 @@ using namespace vespalib; -class MemoryDataStoreTest : public vespalib::TestApp +TEST("testMemoryDataStore") { -private: - void testMemoryDataStore(); - void testVariableSizeVector(); -public: - int Main() override; -}; - -void -MemoryDataStoreTest::testMemoryDataStore() -{ - MemoryDataStore s(alloc::Alloc::alloc(256)); + MemoryDataStore s(alloc::Alloc::alloc(256), nullptr); std::vector<MemoryDataStore::Reference> v; v.push_back(s.push_back("mumbo", 5)); for (size_t i(0); i < 50; i++) { @@ -28,45 +18,9 @@ MemoryDataStoreTest::testMemoryDataStore() v.push_back(s.push_back("mumbo", 5)); EXPECT_EQUAL(52ul, v.size()); EXPECT_NOT_EQUAL(static_cast<const char *>(v[50].data()) + 5, v[51].data()); - for (size_t i(0); i < v.size(); i++) { - EXPECT_EQUAL(0, memcmp("mumbo", v[i].data(), 5)); + for (auto & i : v) { + EXPECT_EQUAL(0, memcmp("mumbo", i.data(), 5)); } } -void -MemoryDataStoreTest::testVariableSizeVector() -{ - VariableSizeVector v(20000, 5*20000); - for (size_t i(0); i < 10000; i++) { - asciistream os; - os << i; - v.push_back(os.str().data(), os.str().size()); - } - for (size_t i(0); i < v.size(); i++) { - asciistream os; - os << i; - EXPECT_EQUAL(os.str().size(), v[i].size()); - EXPECT_EQUAL(0, memcmp(os.str().data(), v[i].data(), os.str().size())); - } - size_t i(0); - for (auto it(v.begin()), mt(v.end()); it != mt; it++, i++) { - asciistream os; - os << i; - EXPECT_EQUAL(os.str().size(), it->size()); - EXPECT_EQUAL(0, memcmp(os.str().data(), (*it).data(), os.str().size())); - } - -} - -int -MemoryDataStoreTest::Main() -{ - TEST_INIT("data_test"); - testMemoryDataStore(); - testVariableSizeVector(); - - TEST_DONE(); -} - -TEST_APPHOOK(MemoryDataStoreTest); - +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/stllike/hash_test.cpp b/vespalib/src/tests/stllike/hash_test.cpp index ae27d2dc58b..dc6e48100a9 100644 --- a/vespalib/src/tests/stllike/hash_test.cpp +++ b/vespalib/src/tests/stllike/hash_test.cpp @@ -494,6 +494,14 @@ TEST("test hash set initializer list - empty") EXPECT_EQUAL(0u, s.size()); } +TEST("empty hash_set can be looked up") +{ + IntHashSet s; + EXPECT_EQUAL(0u, s.size()); + EXPECT_EQUAL(1u, s.capacity()); + EXPECT_TRUE(s.find(1) == s.end()); +} + TEST("test hash set initializer list - 1 element") { IntHashSet s = {1}; diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.h b/vespalib/src/vespa/vespalib/btree/btreeiterator.h index 2418da18c23..9480e5880e0 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.h +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.h @@ -302,22 +302,22 @@ public: /** * Get key at current iterator location. */ - const KeyType & getKey() const { return _leaf.getKey(); } + const KeyType & getKey() const noexcept { return _leaf.getKey(); } /** * Get data at current iterator location. */ - const DataType & getData() const { return _leaf.getData(); } + const DataType & getData() const noexcept { return _leaf.getData(); } /** * Check if iterator is at a valid element, i.e. not at end. */ - bool valid() const { return _leaf.valid(); } + bool valid() const noexcept{ return _leaf.valid(); } /** * Return the number of elements in the tree. */ - size_t size() const; + size_t size() const noexcept; /** @@ -333,7 +333,7 @@ public: /** * Return if the tree has data or not (e.g. keys and data or only keys). */ - static bool hasData() { return LeafNodeType::hasData(); } + static bool hasData() noexcept { return LeafNodeType::hasData(); } /** * Move the iterator directly to end. Used by findHelper method in BTree. diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp index b7927feaa1a..d6dda0047ce 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp @@ -387,15 +387,13 @@ position(uint32_t levels) const res += inode->validLeaves(); for (uint32_t c = elem.getIdx(); c < slots; ++c) { BTreeNode::Ref node = inode->getChild(c); - const InternalNodeType *jnode = - _allocator->mapInternalRef(node); + const InternalNodeType *jnode = _allocator->mapInternalRef(node); res -= jnode->validLeaves(); } } else { for (uint32_t c = 0; c < elem.getIdx(); ++c) { BTreeNode::Ref node = inode->getChild(c); - const InternalNodeType *jnode = - _allocator->mapInternalRef(node); + const InternalNodeType *jnode = _allocator->mapInternalRef(node); res += jnode->validLeaves(); } } @@ -484,7 +482,7 @@ template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> size_t BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: -size() const +size() const noexcept { if (_pathSize > 0) { return _path[_pathSize - 1].getNode()->validLeaves(); diff --git a/vespalib/src/vespa/vespalib/btree/btreenode.h b/vespalib/src/vespa/vespalib/btree/btreenode.h index 0a77a0b4685..4931021d771 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenode.h +++ b/vespalib/src/vespa/vespalib/btree/btreenode.h @@ -67,14 +67,14 @@ public: using Ref = datastore::EntryRef; using ChildRef = datastore::AtomicEntryRef; - bool isLeaf() const { return _level == 0u; } - bool getFrozen() const { return _isFrozen; } - void freeze() { _isFrozen = true; } - void unFreeze() { _isFrozen = false; } - void setLevel(uint8_t level) { _level = level; } - uint32_t getLevel() const { return _level; } - uint32_t validSlots() const { return _validSlots; } - void setValidSlots(uint16_t validSlots_) { _validSlots = validSlots_; } + bool isLeaf() const noexcept { return _level == 0u; } + bool getFrozen() const noexcept { return _isFrozen; } + void freeze() noexcept { _isFrozen = true; } + void unFreeze() noexcept { _isFrozen = false; } + void setLevel(uint8_t level) noexcept { _level = level; } + uint32_t getLevel() const noexcept { return _level; } + uint32_t validSlots() const noexcept { return _validSlots; } + void setValidSlots(uint16_t validSlots_) noexcept { _validSlots = validSlots_; } }; @@ -358,7 +358,7 @@ public: void insert(uint32_t idx, const KeyT & key, BTreeNode::Ref child) { insert(idx, key, BTreeNode::ChildRef(child)); } - uint32_t validLeaves() const { return _validLeaves; } + uint32_t validLeaves() const noexcept { return _validLeaves; } void setValidLeaves(uint32_t newValidLeaves) { _validLeaves = newValidLeaves; } void incValidLeaves(uint32_t delta) { _validLeaves += delta; } void decValidLeaves(uint32_t delta) { _validLeaves -= delta; } diff --git a/vespalib/src/vespa/vespalib/data/memorydatastore.cpp b/vespalib/src/vespa/vespalib/data/memorydatastore.cpp index 354787690c2..6d483e6ff4e 100644 --- a/vespalib/src/vespa/vespalib/data/memorydatastore.cpp +++ b/vespalib/src/vespa/vespalib/data/memorydatastore.cpp @@ -41,21 +41,4 @@ MemoryDataStore::push_back(const void * data, const size_t sz) return ref; } -VariableSizeVector::VariableSizeVector(size_t initialCount, size_t initialBufferSize) - : _vector(), - _store(Alloc::alloc(initialBufferSize)) -{ - _vector.reserve(initialCount); -} - -VariableSizeVector::~VariableSizeVector() = default; - -VariableSizeVector::Reference -VariableSizeVector::push_back(const void * data, const size_t sz) -{ - MemoryDataStore::Reference ptr(_store.push_back(data, sz)); - _vector.push_back(Reference(ptr.data(), sz)); - return _vector.back(); -} - } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/memorydatastore.h b/vespalib/src/vespa/vespalib/data/memorydatastore.h index 7022eb88051..6691211cdd0 100644 --- a/vespalib/src/vespa/vespalib/data/memorydatastore.h +++ b/vespalib/src/vespa/vespalib/data/memorydatastore.h @@ -13,18 +13,19 @@ namespace vespalib { * It has the important property that once an object has been allocated it does not move in memory. * It will start of by allocating one backing buffer and items stored will be appended here. * When limit is exceeded a new buffer is allocated with twice the size of the previous and so it goes. + * You can also provide an optional lock to make it thread safe. **/ class MemoryDataStore { public: class Reference { public: - Reference(void * data_) noexcept : _data(data_) { } + explicit Reference(void * data_) noexcept : _data(data_) { } void * data() noexcept { return _data; } const char * c_str() const noexcept { return static_cast<const char *>(_data); } private: void * _data; }; - MemoryDataStore(alloc::Alloc && initialAlloc=alloc::Alloc::alloc(256), std::mutex * lock=nullptr); + MemoryDataStore(alloc::Alloc && initialAlloc, std::mutex * lock); MemoryDataStore(const MemoryDataStore &) = delete; MemoryDataStore & operator = (const MemoryDataStore &) = delete; ~MemoryDataStore(); @@ -33,7 +34,7 @@ public: * for the lifetime of this object. * @return A pointer/reference to the freshly stored object. */ - Reference push_back(const void * data, const size_t sz); + Reference push_back(const void * data, size_t sz); void swap(MemoryDataStore & rhs) { _buffers.swap(rhs._buffers); } void clear() noexcept { _buffers.clear(); @@ -44,83 +45,5 @@ private: std::mutex * _lock; }; -class VariableSizeVector -{ -public: - class Reference { - public: - Reference(void * data_, size_t sz) noexcept : _data(data_), _sz(sz) { } - void * data() noexcept { return _data; } - const char * c_str() const noexcept { return static_cast<const char *>(_data); } - size_t size() const noexcept { return _sz; } - private: - void * _data; - size_t _sz; - }; - class iterator { - public: - iterator(vespalib::Array<Reference> & v, size_t index) noexcept : _vector(&v), _index(index) {} - Reference & operator * () const noexcept { return (*_vector)[_index]; } - Reference * operator -> () const noexcept { return &(*_vector)[_index]; } - iterator & operator ++ () noexcept { - _index++; - return *this; - } - iterator operator ++ (int) noexcept { - iterator prev = *this; - ++(*this); - return prev; - } - bool operator==(const iterator& rhs) const noexcept { return (_index == rhs._index); } - bool operator!=(const iterator& rhs) const noexcept { return (_index != rhs._index); } - private: - vespalib::Array<Reference> * _vector; - size_t _index; - }; - class const_iterator { - public: - const_iterator(const vespalib::Array<Reference> & v, size_t index) noexcept : _vector(&v), _index(index) {} - const Reference & operator * () const noexcept { return (*_vector)[_index]; } - const Reference * operator -> () const noexcept { return &(*_vector)[_index]; } - const_iterator & operator ++ () noexcept { - _index++; - return *this; - } - const_iterator operator ++ (int) noexcept { - const_iterator prev = *this; - ++(*this); - return prev; - } - bool operator==(const const_iterator& rhs) const noexcept { return (_index == rhs._index); } - bool operator!=(const const_iterator& rhs) const noexcept { return (_index != rhs._index); } - private: - const vespalib::Array<Reference> * _vector; - size_t _index; - }; - VariableSizeVector(const VariableSizeVector &) = delete; - VariableSizeVector & operator = (const VariableSizeVector &) = delete; - VariableSizeVector(size_t initialCount, size_t initialBufferSize); - ~VariableSizeVector(); - iterator begin() noexcept { return iterator(_vector, 0); } - iterator end() noexcept { return iterator(_vector, size()); } - const_iterator begin() const noexcept { return const_iterator(_vector, 0); } - const_iterator end() const noexcept { return const_iterator(_vector, size()); } - Reference push_back(const void * data, const size_t sz); - Reference operator [] (uint32_t index) const noexcept { return _vector[index]; } - size_t size() const noexcept { return _vector.size(); } - bool empty() const noexcept { return _vector.empty(); } - void swap(VariableSizeVector & rhs) noexcept { - _vector.swap(rhs._vector); - _store.swap(rhs._store); - } - void clear() { - _vector.clear(); - _store.clear(); - } -private: - vespalib::Array<Reference> _vector; - MemoryDataStore _store; -}; - } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/slime/named_symbol_lookup.h b/vespalib/src/vespa/vespalib/data/slime/named_symbol_lookup.h index 44dbf05c9da..fb4bc3943ae 100644 --- a/vespalib/src/vespa/vespalib/data/slime/named_symbol_lookup.h +++ b/vespalib/src/vespa/vespalib/data/slime/named_symbol_lookup.h @@ -20,7 +20,7 @@ private: const Memory &_name; public: - NamedSymbolLookup(const SymbolTable &table, const Memory &name) + NamedSymbolLookup(const SymbolTable &table, const Memory &name) noexcept : _table(table), _name(name) {} Symbol lookup() const override; }; diff --git a/vespalib/src/vespa/vespalib/data/slime/slime.cpp b/vespalib/src/vespa/vespalib/data/slime/slime.cpp index d6fac44b360..5a9ec54584d 100644 --- a/vespalib/src/vespa/vespalib/data/slime/slime.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/slime.cpp @@ -6,16 +6,6 @@ namespace vespalib { -Slime::Params::Params() : Params(std::make_unique<SymbolTable>()) { } -Slime::Params::Params(std::unique_ptr<SymbolTable> symbols) noexcept : _symbols(std::move(symbols)), _chunkSize(4096) { } -Slime::Params::Params(Params &&) noexcept = default; -Slime::Params::~Params() = default; - -std::unique_ptr<slime::SymbolTable> -Slime::Params::detachSymbols() { - return std::move(_symbols); -} - Slime::Slime(Params params) : _names(params.detachSymbols()), _stash(std::make_unique<Stash>(params.getChunkSize())), @@ -33,26 +23,6 @@ Slime::reclaimSymbols(Slime &&rhs) { return std::move(rhs._names); } -size_t -Slime::symbols() const noexcept { - return _names->symbols(); -} - -Memory -Slime::inspect(Symbol symbol) const { - return _names->inspect(symbol); -} - -slime::Symbol -Slime::insert(Memory name) { - return _names->insert(name); -} - -slime::Symbol -Slime::lookup(Memory name) const { - return _names->lookup(name); -} - bool operator == (const Slime & a, const Slime & b) noexcept { return a.get() == b.get(); diff --git a/vespalib/src/vespa/vespalib/data/slime/slime.h b/vespalib/src/vespa/vespalib/data/slime/slime.h index a426f906563..4b789838009 100644 --- a/vespalib/src/vespa/vespalib/data/slime/slime.h +++ b/vespalib/src/vespa/vespalib/data/slime/slime.h @@ -21,6 +21,7 @@ #include "symbol.h" #include "symbol_inserter.h" #include "symbol_lookup.h" +#include "symbol_table.h" #include "type.h" #include "value.h" #include "value_factory.h" @@ -51,32 +52,30 @@ private: using Cursor = slime::Cursor; using Inspector = slime::Inspector; - std::unique_ptr<SymbolTable> _names; - std::unique_ptr<Stash> _stash; - RootValue _root; + std::unique_ptr<SymbolTable> _names; + std::unique_ptr<Stash> _stash; + RootValue _root; public: using UP = std::unique_ptr<Slime>; class Params { private: - std::unique_ptr<SymbolTable> _symbols; - size_t _chunkSize; + std::unique_ptr<SymbolTable> _symbols; + size_t _chunkSize; public: - Params(); - explicit Params(std::unique_ptr<SymbolTable> symbols) noexcept; - Params(Params &&) noexcept; - ~Params(); - Params & setChunkSize(size_t chunkSize) { - _chunkSize = chunkSize; - return *this; - } - size_t getChunkSize() const { return _chunkSize; } - std::unique_ptr<SymbolTable> detachSymbols(); + Params() : Params(4096) {} + explicit Params(size_t chunkSize) : _symbols(std::make_unique<SymbolTable>()), _chunkSize(chunkSize) {} + explicit Params(std::unique_ptr<SymbolTable> symbols) noexcept : _symbols(std::move(symbols)), _chunkSize(4096) {} + Params(Params &&) noexcept = default; + ~Params() = default; + size_t getChunkSize() const noexcept { return _chunkSize; } + std::unique_ptr<SymbolTable> detachSymbols() noexcept { return std::move(_symbols); } }; /** * Construct an initially empty Slime object. **/ - explicit Slime(Params params = Params()); + explicit Slime() : Slime(Params()) {} + explicit Slime(Params params); ~Slime(); @@ -88,13 +87,13 @@ public: static std::unique_ptr<SymbolTable> reclaimSymbols(Slime &&rhs); - size_t symbols() const noexcept; + size_t symbols() const noexcept { return _names->symbols(); } - Memory inspect(Symbol symbol) const; + Memory inspect(Symbol symbol) const { return _names->inspect(symbol); } - Symbol insert(Memory name); + Symbol insert(Memory name) { return _names->insert(name); } - Symbol lookup(Memory name) const; + Symbol lookup(Memory name) const { return _names->lookup(name); } Cursor &get() noexcept { return _root.get(); } diff --git a/vespalib/src/vespa/vespalib/data/slime/symbol.h b/vespalib/src/vespa/vespalib/data/slime/symbol.h index 3bce727fad9..a60a49fda27 100644 --- a/vespalib/src/vespa/vespalib/data/slime/symbol.h +++ b/vespalib/src/vespa/vespalib/data/slime/symbol.h @@ -19,8 +19,8 @@ private: public: Symbol() noexcept : _value(UNDEFINED) {} Symbol(uint32_t v) noexcept : _value(v) {} - bool undefined() const { return (_value == UNDEFINED); } - uint32_t getValue() const { return _value; } + bool undefined() const noexcept { return (_value == UNDEFINED); } + uint32_t getValue() const noexcept { return _value; } bool operator<(const Symbol &rhs) const noexcept { return (_value < rhs._value); } bool operator==(const Symbol &rhs) const noexcept { return (_value == rhs._value); } }; diff --git a/vespalib/src/vespa/vespalib/data/slime/symbol_table.cpp b/vespalib/src/vespa/vespalib/data/slime/symbol_table.cpp index a3313516c64..dffe35707fc 100644 --- a/vespalib/src/vespa/vespalib/data/slime/symbol_table.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/symbol_table.cpp @@ -5,10 +5,13 @@ namespace vespalib::slime { -SymbolTable::SymbolTable(size_t expectedNumSymbols) : - _symbols(3*expectedNumSymbols), - _names(expectedNumSymbols, expectedNumSymbols*16) -{ } +SymbolTable::SymbolTable(size_t expectedNumSymbols) + : _stash(), + _symbols(3*expectedNumSymbols), + _names() +{ + _names.reserve(expectedNumSymbols); +} SymbolTable::~SymbolTable() = default; @@ -16,6 +19,7 @@ void SymbolTable::clear() { _names.clear(); _symbols.clear(); + _stash.clear(); } Symbol @@ -23,17 +27,21 @@ SymbolTable::insert(const Memory &name) { SymbolMap::const_iterator pos = _symbols.find(name); if (pos == _symbols.end()) { Symbol symbol(_names.size()); - SymbolVector::Reference r(_names.push_back(name.data, name.size)); - _symbols.insert(std::make_pair(Memory(r.c_str(), r.size()), symbol)); + char *buf = _stash.alloc(name.size); + memcpy(buf, name.data, name.size); + Memory backed(buf, name.size); + _names.push_back(backed); + _symbols.insert(std::make_pair(backed, symbol)); return symbol; } return pos->second; } + Symbol SymbolTable::lookup(const Memory &name) const { SymbolMap::const_iterator pos = _symbols.find(name); if (pos == _symbols.end()) { - return Symbol(); + return {}; } return pos->second; } diff --git a/vespalib/src/vespa/vespalib/data/slime/symbol_table.h b/vespalib/src/vespa/vespalib/data/slime/symbol_table.h index c5f3cf12fd6..0eae65cead0 100644 --- a/vespalib/src/vespa/vespalib/data/slime/symbol_table.h +++ b/vespalib/src/vespa/vespalib/data/slime/symbol_table.h @@ -4,8 +4,8 @@ #include "symbol.h" #include <vespa/vespalib/data/memory.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/vespalib/data/memorydatastore.h> namespace vespalib::slime { @@ -21,21 +21,24 @@ private: } }; using SymbolMap = hash_map<Memory, Symbol, hasher>; - using SymbolVector = VariableSizeVector; + using SymbolVector = std::vector<Memory>; + Stash _stash; SymbolMap _symbols; SymbolVector _names; public: using UP = std::unique_ptr<SymbolTable>; - SymbolTable(size_t expectedNumSymbols=16); + SymbolTable() : SymbolTable(16) {} + explicit SymbolTable(size_t expectedNumSymbols); + SymbolTable(SymbolTable &&) noexcept = default; + SymbolTable & operator=(SymbolTable &&) noexcept = default; ~SymbolTable(); size_t symbols() const noexcept { return _names.size(); } Memory inspect(const Symbol &symbol) const { if (symbol.getValue() > _names.size()) { return Memory(); } - SymbolVector::Reference r(_names[symbol.getValue()]); - return Memory(r.c_str(), r.size()); + return _names[symbol.getValue()]; } Symbol insert(const Memory &name); Symbol lookup(const Memory &name) const; diff --git a/vespalib/src/vespa/vespalib/datastore/compaction_strategy.h b/vespalib/src/vespa/vespalib/datastore/compaction_strategy.h index dd41ed244ae..d3cdc174339 100644 --- a/vespalib/src/vespa/vespalib/datastore/compaction_strategy.h +++ b/vespalib/src/vespa/vespalib/datastore/compaction_strategy.h @@ -2,8 +2,9 @@ #pragma once -#include <iosfwd> +#include <cstddef> #include <cstdint> +#include <iosfwd> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/fuzzy/CMakeLists.txt b/vespalib/src/vespa/vespalib/fuzzy/CMakeLists.txt index 5e8d29980cd..8ccef84d969 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/fuzzy/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_library(vespalib_vespalib_fuzzy OBJECT implicit_levenshtein_dfa.cpp levenshtein_dfa.cpp levenshtein_distance.cpp + table_dfa.cpp unicode_utils.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matching_algorithm.cpp b/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matching_algorithm.cpp index 826b0beffd6..d87afef1fbe 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matching_algorithm.cpp +++ b/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matching_algorithm.cpp @@ -9,6 +9,7 @@ namespace { const vespalib::string brute_force = "brute_force"; const vespalib::string dfa_implicit = "dfa_implicit"; const vespalib::string dfa_explicit = "dfa_explicit"; +const vespalib::string dfa_table = "dfa_table"; } @@ -22,6 +23,8 @@ to_string(FuzzyMatchingAlgorithm algo) return dfa_implicit; case FuzzyMatchingAlgorithm::DfaExplicit: return dfa_explicit; + case FuzzyMatchingAlgorithm::DfaTable: + return dfa_table; default: return ""; } @@ -37,6 +40,8 @@ fuzzy_matching_algorithm_from_string(const vespalib::string& algo, return FuzzyMatchingAlgorithm::DfaImplicit; } else if (algo == dfa_explicit) { return FuzzyMatchingAlgorithm::DfaExplicit; + } else if (algo == dfa_table) { + return FuzzyMatchingAlgorithm::DfaTable; } return default_algo; } diff --git a/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matching_algorithm.h b/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matching_algorithm.h index 83cb121fe5f..9af94e84b89 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matching_algorithm.h +++ b/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matching_algorithm.h @@ -13,7 +13,8 @@ namespace vespalib { enum class FuzzyMatchingAlgorithm { BruteForce, DfaImplicit, - DfaExplicit + DfaExplicit, + DfaTable }; vespalib::string to_string(FuzzyMatchingAlgorithm algo); diff --git a/vespalib/src/vespa/vespalib/fuzzy/inline_tfa.hpp b/vespalib/src/vespa/vespalib/fuzzy/inline_tfa.hpp new file mode 100644 index 00000000000..2475704ec06 --- /dev/null +++ b/vespalib/src/vespa/vespalib/fuzzy/inline_tfa.hpp @@ -0,0 +1,91 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// start of auto-generated code for N = 1 +template <> struct InlineTfa<1> { + static constexpr Transition table[6][8] = { + {{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}}, + {{0,2},{0,2},{0,3},{0,3},{1,1},{1,1},{1,1},{1,1}}, + {{0,0},{0,0},{2,4},{2,4},{1,4},{1,4},{1,2},{1,2}}, + {{0,0},{3,4},{2,4},{2,2},{1,4},{1,5},{1,2},{1,3}}, + {{0,0},{0,0},{0,0},{0,0},{1,4},{1,4},{1,4},{1,4}}, + {{0,0},{3,4},{0,0},{3,4},{1,4},{1,5},{1,4},{1,5}} + }; + static constexpr uint8_t edits[6][3] = { + {2,2,2}, + {0,1,2}, + {1,1,2}, + {1,1,1}, + {1,2,2}, + {1,2,1} + }; +}; +// end of auto-generated code for N = 1 +// start of auto-generated code for N = 2 +template <> struct InlineTfa<2> { + static constexpr Transition table[31][32] = { + {{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}}, + {{0,2},{0,2},{0,2},{0,2},{0,3},{0,3},{0,3},{0,3},{0,4},{0,4},{0,4},{0,4},{0,4},{0,4},{0,4},{0,4},{1,1},{1,1},{1,1},{1,1},{1,1},{1,1},{1,1},{1,1},{1,1},{1,1},{1,1},{1,1},{1,1},{1,1},{1,1},{1,1}}, + {{0,5},{0,5},{0,5},{0,5},{0,6},{0,6},{0,6},{0,6},{0,7},{0,7},{0,7},{0,7},{0,7},{0,7},{0,7},{0,7},{1,8},{1,8},{1,8},{1,8},{1,9},{1,9},{1,9},{1,9},{1,2},{1,2},{1,2},{1,2},{1,2},{1,2},{1,2},{1,2}}, + {{0,5},{0,5},{0,10},{0,10},{0,6},{0,6},{0,11},{0,11},{0,7},{0,7},{0,12},{0,12},{0,7},{0,7},{0,12},{0,12},{1,8},{1,8},{1,13},{1,13},{1,9},{1,9},{1,14},{1,14},{1,2},{1,2},{1,3},{1,3},{1,2},{1,2},{1,3},{1,3}}, + {{0,6},{0,6},{0,11},{0,11},{0,15},{0,15},{0,15},{0,15},{0,7},{0,7},{0,12},{0,12},{0,16},{0,16},{0,16},{0,16},{1,9},{1,9},{1,14},{1,14},{1,17},{1,17},{1,17},{1,17},{1,2},{1,2},{1,3},{1,3},{1,4},{1,4},{1,4},{1,4}}, + {{0,0},{0,0},{0,0},{0,0},{3,18},{3,18},{3,18},{3,18},{2,18},{2,18},{2,18},{2,18},{2,19},{2,19},{2,19},{2,19},{1,18},{1,18},{1,18},{1,18},{1,20},{1,20},{1,20},{1,20},{1,19},{1,19},{1,19},{1,19},{1,5},{1,5},{1,5},{1,5}}, + {{0,0},{0,0},{4,18},{4,18},{3,18},{3,18},{3,19},{3,19},{2,18},{2,18},{2,20},{2,20},{2,19},{2,19},{2,5},{2,5},{1,18},{1,18},{1,21},{1,21},{1,20},{1,20},{1,22},{1,22},{1,19},{1,19},{1,23},{1,23},{1,5},{1,5},{1,6},{1,6}}, + {{2,19},{2,19},{2,5},{2,5},{3,8},{3,8},{3,8},{3,8},{2,19},{2,19},{2,5},{2,5},{3,8},{3,8},{3,8},{3,8},{1,5},{1,5},{1,6},{1,6},{1,7},{1,7},{1,7},{1,7},{1,5},{1,5},{1,6},{1,6},{1,7},{1,7},{1,7},{1,7}}, + {{0,19},{0,19},{0,19},{0,19},{0,19},{0,19},{0,19},{0,19},{0,5},{0,5},{0,5},{0,5},{0,5},{0,5},{0,5},{0,5},{1,8},{1,8},{1,8},{1,8},{1,8},{1,8},{1,8},{1,8},{1,8},{1,8},{1,8},{1,8},{1,8},{1,8},{1,8},{1,8}}, + {{0,19},{0,19},{0,19},{0,19},{0,23},{0,23},{0,23},{0,23},{0,5},{0,5},{0,5},{0,5},{0,6},{0,6},{0,6},{0,6},{1,8},{1,8},{1,8},{1,8},{1,9},{1,9},{1,9},{1,9},{1,8},{1,8},{1,8},{1,8},{1,9},{1,9},{1,9},{1,9}}, + {{0,0},{5,18},{0,0},{5,18},{3,18},{3,20},{3,18},{3,20},{2,18},{2,21},{2,18},{2,21},{2,19},{2,23},{2,19},{2,23},{1,18},{1,24},{1,18},{1,24},{1,20},{1,25},{1,20},{1,25},{1,19},{1,26},{1,19},{1,26},{1,5},{1,10},{1,5},{1,10}}, + {{0,0},{5,18},{4,18},{4,19},{3,18},{3,20},{3,19},{3,5},{2,18},{2,21},{2,20},{2,22},{2,19},{2,23},{2,5},{2,6},{1,18},{1,24},{1,21},{1,27},{1,20},{1,25},{1,22},{1,28},{1,19},{1,26},{1,23},{1,29},{1,5},{1,10},{1,6},{1,11}}, + {{2,19},{2,23},{2,5},{2,6},{3,8},{3,9},{3,8},{3,9},{2,19},{2,23},{2,5},{2,6},{3,8},{3,9},{3,8},{3,9},{1,5},{1,10},{1,6},{1,11},{1,7},{1,12},{1,7},{1,12},{1,5},{1,10},{1,6},{1,11},{1,7},{1,12},{1,7},{1,12}}, + {{0,19},{0,19},{0,26},{0,26},{0,19},{0,19},{0,26},{0,26},{0,5},{0,5},{0,10},{0,10},{0,5},{0,5},{0,10},{0,10},{1,8},{1,8},{1,13},{1,13},{1,8},{1,8},{1,13},{1,13},{1,8},{1,8},{1,13},{1,13},{1,8},{1,8},{1,13},{1,13}}, + {{0,19},{0,19},{0,26},{0,26},{0,23},{0,23},{0,29},{0,29},{0,5},{0,5},{0,10},{0,10},{0,6},{0,6},{0,11},{0,11},{1,8},{1,8},{1,13},{1,13},{1,9},{1,9},{1,14},{1,14},{1,8},{1,8},{1,13},{1,13},{1,9},{1,9},{1,14},{1,14}}, + {{3,19},{3,5},{4,8},{4,8},{3,19},{3,5},{4,8},{4,8},{2,5},{2,6},{2,7},{2,7},{2,5},{2,6},{2,7},{2,7},{1,22},{1,28},{1,30},{1,30},{1,22},{1,28},{1,30},{1,30},{1,6},{1,11},{1,15},{1,15},{1,6},{1,11},{1,15},{1,15}}, + {{2,5},{2,6},{2,7},{2,7},{3,8},{3,9},{3,2},{3,2},{2,5},{2,6},{2,7},{2,7},{3,8},{3,9},{3,2},{3,2},{1,6},{1,11},{1,15},{1,15},{1,7},{1,12},{1,16},{1,16},{1,6},{1,11},{1,15},{1,15},{1,7},{1,12},{1,16},{1,16}}, + {{0,6},{0,6},{0,11},{0,11},{0,15},{0,15},{0,15},{0,15},{0,6},{0,6},{0,11},{0,11},{0,15},{0,15},{0,15},{0,15},{1,9},{1,9},{1,14},{1,14},{1,17},{1,17},{1,17},{1,17},{1,9},{1,9},{1,14},{1,14},{1,17},{1,17},{1,17},{1,17}}, + {{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18}}, + {{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},{2,18},{2,18},{2,18},{2,18},{2,18},{2,18},{2,18},{2,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,18},{1,19},{1,19},{1,19},{1,19},{1,19},{1,19},{1,19},{1,19}}, + {{0,0},{0,0},{0,0},{0,0},{3,18},{3,18},{3,18},{3,18},{0,0},{0,0},{0,0},{0,0},{3,18},{3,18},{3,18},{3,18},{1,18},{1,18},{1,18},{1,18},{1,20},{1,20},{1,20},{1,20},{1,18},{1,18},{1,18},{1,18},{1,20},{1,20},{1,20},{1,20}}, + {{0,0},{0,0},{4,18},{4,18},{0,0},{0,0},{4,18},{4,18},{0,0},{0,0},{4,18},{4,18},{0,0},{0,0},{4,18},{4,18},{1,18},{1,18},{1,21},{1,21},{1,18},{1,18},{1,21},{1,21},{1,18},{1,18},{1,21},{1,21},{1,18},{1,18},{1,21},{1,21}}, + {{0,0},{0,0},{4,18},{4,18},{3,18},{3,18},{3,19},{3,19},{0,0},{0,0},{4,18},{4,18},{3,18},{3,18},{3,19},{3,19},{1,18},{1,18},{1,21},{1,21},{1,20},{1,20},{1,22},{1,22},{1,18},{1,18},{1,21},{1,21},{1,20},{1,20},{1,22},{1,22}}, + {{0,0},{0,0},{4,18},{4,18},{0,0},{0,0},{4,18},{4,18},{2,18},{2,18},{2,20},{2,20},{2,18},{2,18},{2,20},{2,20},{1,18},{1,18},{1,21},{1,21},{1,18},{1,18},{1,21},{1,21},{1,19},{1,19},{1,23},{1,23},{1,19},{1,19},{1,23},{1,23}}, + {{0,0},{5,18},{0,0},{5,18},{0,0},{5,18},{0,0},{5,18},{0,0},{5,18},{0,0},{5,18},{0,0},{5,18},{0,0},{5,18},{1,18},{1,24},{1,18},{1,24},{1,18},{1,24},{1,18},{1,24},{1,18},{1,24},{1,18},{1,24},{1,18},{1,24},{1,18},{1,24}}, + {{0,0},{5,18},{0,0},{5,18},{3,18},{3,20},{3,18},{3,20},{0,0},{5,18},{0,0},{5,18},{3,18},{3,20},{3,18},{3,20},{1,18},{1,24},{1,18},{1,24},{1,20},{1,25},{1,20},{1,25},{1,18},{1,24},{1,18},{1,24},{1,20},{1,25},{1,20},{1,25}}, + {{0,0},{5,18},{0,0},{5,18},{0,0},{5,18},{0,0},{5,18},{2,18},{2,21},{2,18},{2,21},{2,18},{2,21},{2,18},{2,21},{1,18},{1,24},{1,18},{1,24},{1,18},{1,24},{1,18},{1,24},{1,19},{1,26},{1,19},{1,26},{1,19},{1,26},{1,19},{1,26}}, + {{0,0},{5,18},{4,18},{4,19},{0,0},{5,18},{4,18},{4,19},{0,0},{5,18},{4,18},{4,19},{0,0},{5,18},{4,18},{4,19},{1,18},{1,24},{1,21},{1,27},{1,18},{1,24},{1,21},{1,27},{1,18},{1,24},{1,21},{1,27},{1,18},{1,24},{1,21},{1,27}}, + {{0,0},{5,18},{4,18},{4,19},{3,18},{3,20},{3,19},{3,5},{0,0},{5,18},{4,18},{4,19},{3,18},{3,20},{3,19},{3,5},{1,18},{1,24},{1,21},{1,27},{1,20},{1,25},{1,22},{1,28},{1,18},{1,24},{1,21},{1,27},{1,20},{1,25},{1,22},{1,28}}, + {{0,0},{5,18},{4,18},{4,19},{0,0},{5,18},{4,18},{4,19},{2,18},{2,21},{2,20},{2,22},{2,18},{2,21},{2,20},{2,22},{1,18},{1,24},{1,21},{1,27},{1,18},{1,24},{1,21},{1,27},{1,19},{1,26},{1,23},{1,29},{1,19},{1,26},{1,23},{1,29}}, + {{3,19},{3,5},{4,8},{4,8},{3,19},{3,5},{4,8},{4,8},{3,19},{3,5},{4,8},{4,8},{3,19},{3,5},{4,8},{4,8},{1,22},{1,28},{1,30},{1,30},{1,22},{1,28},{1,30},{1,30},{1,22},{1,28},{1,30},{1,30},{1,22},{1,28},{1,30},{1,30}} + }; + static constexpr uint8_t edits[31][5] = { + {3,3,3,3,3}, + {0,1,2,3,3}, + {1,1,2,3,3}, + {1,1,2,2,3}, + {1,1,1,2,3}, + {2,2,2,3,3}, + {2,2,2,2,3}, + {2,2,1,2,3}, + {1,2,3,3,3}, + {1,2,2,3,3}, + {2,2,2,3,2}, + {2,2,2,2,2}, + {2,2,1,2,2}, + {1,2,3,2,3}, + {1,2,2,2,3}, + {2,2,2,1,2}, + {2,2,1,1,2}, + {1,2,1,2,3}, + {2,3,3,3,3}, + {2,2,3,3,3}, + {2,3,2,3,3}, + {2,3,3,2,3}, + {2,3,2,2,3}, + {2,2,3,2,3}, + {2,3,3,3,2}, + {2,3,2,3,2}, + {2,2,3,3,2}, + {2,3,3,2,2}, + {2,3,2,2,2}, + {2,2,3,2,2}, + {2,3,2,1,2} + }; +}; +// end of auto-generated code for N = 2 diff --git a/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.cpp b/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.cpp index 1caae408176..5f6d0ae9956 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.cpp +++ b/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "explicit_levenshtein_dfa.h" #include "implicit_levenshtein_dfa.h" +#include "table_dfa.h" #include "levenshtein_dfa.h" #include "unicode_utils.h" #include <vespa/vespalib/util/stringfmt.h> @@ -53,14 +54,19 @@ LevenshteinDfa LevenshteinDfa::build(std::string_view target_string, uint8_t max } else { // max_edits == 2 return LevenshteinDfa(std::make_unique<ImplicitLevenshteinDfa<FixedMaxEditDistanceTraits<2>>>(std::move(target_string_u32), is_cased)); } - } else { // DfaType::Explicit + } else if(dfa_type == DfaType::Explicit) { if (max_edits == 1) { return ExplicitLevenshteinDfaBuilder<FixedMaxEditDistanceTraits<1>>(std::move(target_string_u32), is_cased).build_dfa(); } else { // max_edits == 2 return ExplicitLevenshteinDfaBuilder<FixedMaxEditDistanceTraits<2>>(std::move(target_string_u32), is_cased).build_dfa(); } + } else { // DfaType::Table + if (max_edits == 1) { + return LevenshteinDfa(std::make_unique<TableDfa<1>>(std::move(target_string_u32), is_cased)); + } else { // max_edits == 2 + return LevenshteinDfa(std::make_unique<TableDfa<2>>(std::move(target_string_u32), is_cased)); + } } - } LevenshteinDfa LevenshteinDfa::build(std::string_view target_string, uint8_t max_edits, Casing casing) { @@ -87,9 +93,11 @@ std::ostream& operator<<(std::ostream& os, const LevenshteinDfa::MatchResult& mo std::ostream& operator<<(std::ostream& os, LevenshteinDfa::DfaType dt) { if (dt == LevenshteinDfa::DfaType::Implicit) { os << "Implicit"; - } else { - assert(dt == LevenshteinDfa::DfaType::Explicit); + } else if (dt == LevenshteinDfa::DfaType::Explicit) { os << "Explicit"; + } else { + assert(dt == LevenshteinDfa::DfaType::Table); + os << "Table"; } return os; } diff --git a/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.h b/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.h index c6ca06d4de3..1652631e968 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.h +++ b/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.h @@ -58,8 +58,8 @@ namespace vespalib::fuzzy { * ====== Unicode support ====== * * Matching and successor generation is fully Unicode-aware. All input strings are expected - * to be in UTF-8, and the generated successor is also encoded as UTF-8 (with some caveats; - * see the documentation for match()). + * to be in UTF-8, and the generated successor is encoded as UTF-8 (with some caveats; see + * the documentation for match()) or UTF-32, depending on the chosen `match()` overload. * * Internally, matching is done on UTF-32 code points and the DFA itself is built around * UTF-32. This is unlike Lucene, which converts a UTF-32 DFA to an equivalent UTF-8 DFA. @@ -159,7 +159,7 @@ public: /** * Attempts to match the source string `source` with the target string this DFA was - * built with, emitting a successor string on mismatch if `successor_out` != nullptr. + * built with. * * `source` must not contain any null UTF-8 chars. * @@ -179,13 +179,23 @@ public: * Attempts to match the source string `source` with the target string this DFA was * built with, emitting a successor string into `successor_out` on mismatch. * - * See `match(source)` for semantics of returned MatchResult. + * In the case of a _match_, the following holds: + * + * - The returned MatchResult has the same semantics as `match(source)`. + * - `successor_out` has a _prefix_ equal to its value that was originally passed + * in at the time of match() being called. The _suffix_ of the string is unspecified, + * i.e. it may or may not have been modified. * * In the case of a _mismatch_, the following holds: * - * - `successor_out` is modified to contain the next (in byte-wise ordering) possible - * _matching_ string S so that there exists no other matching string S' that is - * greater than `source` but smaller than S. + * - `successor_out` has a _prefix_ equal to its value that was originally passed + * in at the time of match() being called. + * - `successor_out` has a _suffix_ that contains the next (in byte-wise ordering) + * possible _matching_ string S so that there exists no other matching string S' + * that is greater than `source` but smaller than S. + * The caller must explicitly be aware of any prefixes it sends in, as it is + * entirely ignored for the purposes of ordering the successor string vis-a-vis + * the input source string. * - `successor_out` contains UTF-8 bytes that are within what UTF-8 can legally * encode in bitwise form, but the _code points_ they encode may not be valid. * In particular, surrogate pair ranges and U+10FFFF+1 may be encoded, neither of @@ -203,12 +213,8 @@ public: * is what is passed to the DFA match() function. * * Memory allocation: - * This function does not directly or indirectly allocate any heap memory if either: - * - * - the input string is within the max edit distance, or - * - `successor_out` is nullptr, or - * - `successor_out` has sufficient capacity to hold the generated successor - * + * This function does not directly or indirectly allocate any heap memory if the + * `successor_out` string provided is large enough to fit any generated successor. * By reusing the successor string across many calls, this therefore amortizes memory * allocations down to near zero per invocation. */ @@ -220,7 +226,8 @@ public: * internally, and is therefore expected to be more efficient. * * The code point ordering of the UTF-32 successor string is identical to that its UTF-8 - * equivalent. + * equivalent. This includes the special cases where the successor may contain code points + * outside the legal Unicode range. */ [[nodiscard]] MatchResult match(std::string_view source, std::vector<uint32_t>& successor_out) const; @@ -231,7 +238,8 @@ public: enum class DfaType { Implicit, - Explicit + Explicit, + Table }; /** diff --git a/vespalib/src/vespa/vespalib/fuzzy/match_algorithm.hpp b/vespalib/src/vespa/vespalib/fuzzy/match_algorithm.hpp index fb5ec32abc7..9f5dc0fb0c0 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/match_algorithm.hpp +++ b/vespalib/src/vespa/vespalib/fuzzy/match_algorithm.hpp @@ -136,15 +136,14 @@ struct MatchAlgorithm { * that has nothing in common with the source altogether. * Example: "gp" -> "hfood" (+1 char value case) * - * Performance note: - * Both the input and successor output strings are in UTF-8 format. To avoid doing - * duplicate work, we keep track of the byte length of the string prefix that will be - * part of the successor and simply copy it verbatim instead of building the string - * from converted UTF-32 -> UTF-8 chars as we go. This optimization cannot be used - * when one or more of the prefix characters have been lowercase-transformed. + * Note for cased vs. uncased matching: when uncased matching is specified, we always + * match "as if" both the target and source strings are lowercased. This means that + * successor strings are generated based on this form, _not_ on the original form. + * Example: uncased matching for target "food" with input "FOXX". This generates the + * successor "foyd" (and _not_ "FOyd"), as the latter would imply a completely different + * ordering when compared byte-wise against an implicitly lowercased dictionary. * * TODO let matcher know if source string is pre-normalized (i.e. lowercased). - * TODO consider opportunistically appending prefix as we go instead of only when needed. */ template <DfaMatcher Matcher, typename SuccessorT> static MatchResult match(const Matcher& matcher, @@ -153,22 +152,19 @@ struct MatchAlgorithm { { using StateType = typename Matcher::StateType; Utf8Reader u8_reader(source.data(), source.size()); - uint32_t n_prefix_u8_bytes = 0; + uint32_t n_prefix_chars = static_cast<uint32_t>(successor_out.size()); // Don't touch any existing prefix uint32_t char_after_prefix = 0; StateType last_state_with_higher_out = StateType{}; - bool can_use_raw_prefix = true; StateType state = matcher.start(); while (u8_reader.hasMore()) { - const auto u8_pos_before_char = u8_reader.getPos(); - const uint32_t raw_mch = u8_reader.getChar(); - const uint32_t mch = normalized_match_char(raw_mch, matcher.is_cased()); - if (raw_mch != mch) { - can_use_raw_prefix = false; // FIXME this is pessimistic; considers entire string, not just prefix - } + const auto pos_before_char = static_cast<uint32_t>(successor_out.size()); + const uint32_t raw_mch = u8_reader.getChar(); + const uint32_t mch = normalized_match_char(raw_mch, matcher.is_cased()); + append_utf32_char(successor_out, mch); if (matcher.has_higher_out_edge(state, mch)) { last_state_with_higher_out = state; - n_prefix_u8_bytes = u8_pos_before_char; + n_prefix_chars = pos_before_char; char_after_prefix = mch; } auto maybe_next = matcher.match_input(state, mch); @@ -176,8 +172,7 @@ struct MatchAlgorithm { state = maybe_next; } else { // Can never match; find the successor - emit_successor_prefix(successor_out, source, n_prefix_u8_bytes, - matcher.is_cased() || can_use_raw_prefix); + successor_out.resize(n_prefix_chars); // Always <= successor_out.size() assert(matcher.valid_state(last_state_with_higher_out)); backtrack_and_emit_greater_suffix(matcher, last_state_with_higher_out, char_after_prefix, successor_out); @@ -188,8 +183,7 @@ struct MatchAlgorithm { if (edits <= max_edits()) { return MatchResult::make_match(max_edits(), edits); } - emit_successor_prefix(successor_out, source, source.size(), - matcher.is_cased() || can_use_raw_prefix); + // Successor prefix already filled, just need to emit the suffix emit_smallest_matching_suffix(matcher, state, successor_out); return MatchResult::make_mismatch(max_edits()); } @@ -320,48 +314,6 @@ struct MatchAlgorithm { } } - template <typename T> - static constexpr bool has_8bit_value_type() noexcept { - return sizeof(typename T::value_type) == 1; - } - - /** - * The successor prefix is the prefix of the source string up to (but not including) the - * point where we emit a lexicographically higher character. Ideally we can just copy the - * UTF-8 bytes verbatim from the source into the successor. This is possible when one of - * the following holds: - * - * - DFA uses Cased (i.e. exact) matching, or - * - DFA uses Uncased, but none of the characters in the prefix triggered a lowercase - * transform. This means the prefix is already as-if lowercased, and we can copy it - * verbatim. - * - * In the case that we can't copy verbatim, we currently have to explicitly normalize the - * prefix by converting it to its lowercased form. - * - * Example: Uncased matching for target "food" with input "FOXX". This generates the - * successor "foyd" (and _not_ "FOyd"), as the latter would imply a completely different - * ordering when compared byte-wise against an implicitly lowercased dictionary. - */ - template <typename SuccessorT> - static void emit_successor_prefix(SuccessorT& successor_out, std::string_view source, - uint32_t n_prefix_u8_bytes, bool emit_raw_prefix_u8_bytes) - { - // TODO redesign prefix output wiring - if constexpr (has_8bit_value_type<SuccessorT>()) { - if (emit_raw_prefix_u8_bytes) { - successor_out = source.substr(0, n_prefix_u8_bytes); - return; - } - } - // TODO avoid duplicate work...! :I - successor_out.clear(); - Utf8Reader u8_reader(source.data(), source.size()); - while (u8_reader.getPos() < n_prefix_u8_bytes) { - append_utf32_char(successor_out, LowerCase::convert(u8_reader.getChar())); - } - } - static uint32_t normalized_match_char(uint32_t in_ch, bool is_cased) noexcept { return (is_cased ? in_ch : LowerCase::convert(in_ch)); } diff --git a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h index d20cfc07a9a..dfec0bac4a8 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h +++ b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h @@ -112,11 +112,11 @@ std::ostream& operator<<(std::ostream& os, const FixedSparseState<MaxEdits>& s) if (i != 0) { os << ", "; } - for (size_t j = last_idx; j < s.indices[i]; ++j) { + for (size_t j = last_idx; j < s.index(i); ++j) { os << "-, "; } - last_idx = s.indices[i] + 1; - os << static_cast<uint32_t>(s.costs[i]); + last_idx = s.index(i) + 1; + os << static_cast<uint32_t>(s.cost(i)); } os << "]"; return os; diff --git a/vespalib/src/vespa/vespalib/fuzzy/table_dfa.cpp b/vespalib/src/vespa/vespalib/fuzzy/table_dfa.cpp new file mode 100644 index 00000000000..943349818fb --- /dev/null +++ b/vespalib/src/vespa/vespalib/fuzzy/table_dfa.cpp @@ -0,0 +1,10 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "table_dfa.hpp" + +namespace vespalib::fuzzy { + +template class TableDfa<1>; +template class TableDfa<2>; + +} diff --git a/vespalib/src/vespa/vespalib/fuzzy/table_dfa.h b/vespalib/src/vespa/vespalib/fuzzy/table_dfa.h new file mode 100644 index 00000000000..45c52c84bad --- /dev/null +++ b/vespalib/src/vespa/vespalib/fuzzy/table_dfa.h @@ -0,0 +1,63 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "levenshtein_dfa.h" +#include <vector> + +namespace vespalib::fuzzy { + +/** + * This implementation is based on the paper 'Fast string correction + * with Levenshtein automata' from 2002 by Klaus U. Schulz and Stoyan + * Mihov. + * + * Given the maximal distance N, a generic parameterized transition + * table is calculated up-front. When a specific word is given, a + * simple lookup structure is created to enumerate the possible + * characteristic vectors for each position in the given + * word. Together, these structures can be used to simulate the + * traversal of a hypothetical Levenshtein dfa that will never be + * created. + * + * Approaching the end of the word is handled by padding the + * characteristic vectors with 0 bits for everything after the word + * ends. In addition, a unit test verifies that there is no possible + * sequence of events that leads to the minimal boundary of the state + * exceeding the boundary of the word itself. This means that the + * simulated dfa can be stepped freely without checking for word size. + **/ +template <uint8_t N> +class TableDfa final : public LevenshteinDfa::Impl +{ +public: + // characteristic vector for a specific input value indicating how + // it matches the window starting at the minimal boundary. + struct CV { + uint32_t input; + uint32_t match; + CV() noexcept : input(0), match(0) {} + }; + static constexpr size_t window_size() { return 2 * N + 1; } + struct Lookup { + std::array<CV, window_size()> list; + Lookup() noexcept : list() {} + }; + +private: + const std::vector<Lookup> _lookup; + const bool _is_cased; + + static std::vector<Lookup> make_lookup(const std::vector<uint32_t> &str); + +public: + using MatchResult = LevenshteinDfa::MatchResult; + TableDfa(std::vector<uint32_t> str, bool is_cased); + ~TableDfa() override; + [[nodiscard]] MatchResult match(std::string_view source) const override; + [[nodiscard]] MatchResult match(std::string_view source, std::string& successor_out) const override; + [[nodiscard]] MatchResult match(std::string_view source, std::vector<uint32_t>& successor_out) const override; + [[nodiscard]] size_t memory_usage() const noexcept override; + void dump_as_graphviz(std::ostream& os) const override; +}; + +} diff --git a/vespalib/src/vespa/vespalib/fuzzy/table_dfa.hpp b/vespalib/src/vespa/vespalib/fuzzy/table_dfa.hpp new file mode 100644 index 00000000000..de850681113 --- /dev/null +++ b/vespalib/src/vespa/vespalib/fuzzy/table_dfa.hpp @@ -0,0 +1,586 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "table_dfa.h" +#include "match_algorithm.hpp" +#include <vespa/vespalib/util/stringfmt.h> +#include <cassert> +#include <stdexcept> +#include <algorithm> +#include <map> +#include <ostream> +#include <set> +#include <queue> + +namespace vespalib::fuzzy { + +namespace { + +using vespalib::make_string_short::fmt; + +// It is useful to know the number of states compile time to be able +// to pack lookup tables better. +template <uint8_t N> constexpr size_t num_states(); +template <> constexpr size_t num_states<1>() { return 6; } +template <> constexpr size_t num_states<2>() { return 31; } +template <> constexpr size_t num_states<3>() { return 197; } + +template <uint8_t N> constexpr size_t window_size() { return 2 * N + 1; } +template <uint8_t N> constexpr size_t num_transitions() { return 1 << window_size<N>(); } + + +auto diff(auto a, auto b) { return (a > b) ? (a - b) : (b - a); } + +// A Position combines an index into a word being matched with the +// number of edits needed to get there. This maps directly onto a +// specific state in the NFA used to match a word. Note that the sort +// order prefers low edits over low indexs. This is to ensure that a +// position that subsumes another position will always sort before it. +struct Position { + uint32_t index; + uint32_t edits; + Position(uint32_t index_in, uint32_t edits_in) noexcept + : index(index_in), edits(edits_in) {} + static Position start() noexcept { return Position(0,0); } + bool subsumes(const Position &rhs) const noexcept { + if (edits >= rhs.edits) { + return false; + } + return diff(index, rhs.index) <= (rhs.edits - edits); + } + Position materialize(uint32_t target_index) const noexcept { + return Position(target_index, edits + diff(index, target_index)); + } + bool operator==(const Position &rhs) const noexcept { + return (index == rhs.index) && (edits == rhs.edits); + } + bool operator<(const Position &rhs) const noexcept { + return std::tie(edits,index) < std::tie(rhs.edits, rhs.index); + } + template <uint8_t N> + void add_elementary_transitions(const std::vector<bool> &bits, std::vector<Position> &dst) const { + assert(bits.size() > index); + if (!bits[index]) { + dst.emplace_back(index, edits + 1); + dst.emplace_back(index + 1, edits + 1); + } + for (uint32_t e = 0; (edits + e) <= N; ++e) { + assert(bits.size() > (index + e)); + if (bits[index + e]) { + dst.emplace_back(index + e + 1, edits + e); + } + } + } + vespalib::string to_string() const { return fmt("%u#%u", index, edits); } +}; + +// A State is a collection of different Positions that do not subsume +// each other. If the minimal boundary of a state is larger than 0, it +// can be lifted from the state in a normalizing operation that will +// renumber the position indexes such that the minimal boundary of the +// state becomes 0. This is to allow parameterized states where the +// general progress of matching the string (minimal boundary of +// non-normalized state) is untangled from the local competing edit +// alternatives (normalized state). +struct State { + std::vector<Position> list; + State() noexcept : list() {} + static State failed() noexcept { return State(); } + static State start() { + State result; + result.list.push_back(Position::start()); + return result; + } + bool operator<(const State &rhs) const { + return list < rhs.list; + } + uint32_t minimal_boundary() const noexcept { + if (list.empty()) { + return 0; + } + uint32_t min = list[0].index; + for (size_t i = 1; i < list.size(); ++i) { + min = std::min(min, list[i].index); + } + return min; + } + uint32_t normalize() { + uint32_t min = minimal_boundary(); + if (min > 0) { + for (auto &entry: list) { + entry.index -= min; + } + } + return min; + } + template <uint8_t N> + static State create(std::vector<Position> list_in) { + State result; + auto want = [&result](Position pos) { + if (pos.edits > N) { + return false; + } + for (const auto &old_pos: result.list) { + if (old_pos == pos || old_pos.subsumes(pos)) { + return false; + } + } + return true; + }; + std::sort(list_in.begin(), list_in.end()); + for (const auto &pos: list_in) { + if (want(pos)) { + result.list.push_back(pos); + } + } + return result; + } + template <uint8_t N> + State next(const std::vector<bool> &bits) const { + std::vector<Position> tmp; + for (const auto &pos: list) { + pos.add_elementary_transitions<N>(bits, tmp); + } + return create<N>(std::move(tmp)); + } + template <uint8_t N> + std::vector<uint8_t> make_edit_vector() const { + std::vector<uint8_t> result(window_size<N>(), N + 1); + for (const auto &pos: list) { + for (uint32_t i = 0; i < window_size<N>(); ++i) { + result[i] = std::min(result[i], uint8_t(pos.materialize(i).edits)); + } + } + return result; + } + vespalib::string to_string() const { + vespalib::string result = "{"; + for (size_t i = 0; i < list.size(); ++i) { + if (i > 0) { + result.append(","); + } + result.append(list[i].to_string()); + } + result.append("}"); + return result; + } +}; + +// Keeps track of unique states, assigning an integer value to each +// state. Only states with minimal boundary 0 is allowed to be +// inserted into a state repo. Each repo is seeded with the empty +// state (0) and the start state (1). An assigned integer value can be +// mapped back into the originating state. +struct StateRepo { + using Map = std::map<State,uint32_t>; + using Ref = Map::iterator; + Map seen; + std::vector<Ref> refs; + StateRepo() noexcept : seen(), refs() { + auto failed_idx = state_to_idx(State::failed()); + auto start_idx = state_to_idx(State::start()); + assert(failed_idx == 0); + assert(start_idx == 1); + } + ~StateRepo(); + size_t size() const { return seen.size(); } + uint32_t state_to_idx(const State &state) { + assert(state.minimal_boundary() == 0); + uint32_t next = refs.size(); + auto [pos, inserted] = seen.emplace(state, next); + if (inserted) { + refs.push_back(pos); + } + assert(seen.size() == refs.size()); + return pos->second; + } + const State &idx_to_state(uint32_t idx) const { + assert(idx < refs.size()); + return refs[idx]->first; + } +}; +[[maybe_unused]] StateRepo::~StateRepo() = default; + +template <uint8_t N> +std::vector<bool> expand_bits(uint32_t value) { + static_assert(N < 10); + std::vector<bool> result(window_size<N>()); + uint32_t look_for = num_transitions<N>(); + assert(value < look_for); + for (size_t i = 0; i < result.size(); ++i) { + look_for >>= 1; + result[i] = (value & look_for); + } + return result; +} + +template <uint8_t N> +StateRepo make_state_repo() { + StateRepo repo; + for (uint32_t idx = 0; idx < repo.size(); ++idx) { + const State &state = repo.idx_to_state(idx); + for (uint32_t i = 0; i < num_transitions<N>(); ++i) { + State new_state = state.next<N>(expand_bits<N>(i)); + (void) new_state.normalize(); + (void) repo.state_to_idx(new_state); + } + } + return repo; +} + +struct Transition { + uint8_t step; + uint8_t state; + constexpr Transition() noexcept : step(0), state(0) {} + constexpr Transition(uint8_t di, uint8_t ns) noexcept : step(di), state(ns) {} +}; + +template <uint8_t N> struct InlineTfa; +#include "inline_tfa.hpp" + +template <uint8_t N> +struct Tfa { + // what happens when following a transition from a state? + std::array<std::array<Transition,num_transitions<N>()>,num_states<N>()> table; + + // how many edits did we use to match the target word? + std::array<std::array<uint8_t,window_size<N>()>,num_states<N>()> edits; +}; + +template <uint8_t N> +std::unique_ptr<Tfa<N>> make_tfa() { + auto tfa = std::make_unique<Tfa<N>>(); + StateRepo repo; + uint32_t state_idx = 0; + for (; state_idx < repo.size(); ++state_idx) { + const State &state = repo.idx_to_state(state_idx); + for (uint32_t i = 0; i < num_transitions<N>(); ++i) { + State new_state = state.next<N>(expand_bits<N>(i)); + uint32_t step = new_state.normalize(); + uint32_t new_state_idx = repo.state_to_idx(new_state); + assert(step < 256); + assert(new_state_idx < 256); + tfa->table[state_idx][i].step = step; + tfa->table[state_idx][i].state = new_state_idx; + } + auto edits = state.make_edit_vector<N>(); + assert(edits.size() == window_size<N>()); + for (uint32_t i = 0; i < window_size<N>(); ++i) { + tfa->edits[state_idx][i] = edits[i]; + } + } + assert(repo.size() == num_states<N>()); + assert(state_idx == num_states<N>()); + return tfa; +} + +template <typename T> +vespalib::string format_vector(const std::vector<T> &vector, bool compact = false) { + vespalib::string str = compact ? "" : "["; + for (size_t i = 0; i < vector.size(); ++i) { + if (i > 0 && !compact) { + str.append(","); + } + str.append(fmt("%u", uint32_t(vector[i]))); + } + if (!compact) { + str.append("]"); + } + return str; +} + +// A table-based state using the InlineTfa tables to simulate stepping +// a dfa with max edit distance N. The state itself is represented by +// a number used as offset into these tables (state). Since the state +// is parametric, we also store the minimal boundary of the state +// separately (index). +template <uint8_t N> +struct TfaState { + uint32_t index; + uint32_t state; + // needed by dfa matcher concept (should use std::declval instead) + constexpr TfaState() noexcept : index(0), state(0) {} + constexpr TfaState(uint32_t i, uint32_t s) noexcept : index(i), state(s) {} + static constexpr TfaState start() { return TfaState(0, 1); } + constexpr bool valid() const noexcept { return state != 0; } + constexpr TfaState next(uint32_t bits) const noexcept { + const auto &entry = InlineTfa<N>::table[state][bits]; + return TfaState(index + entry.step, entry.state); + } + constexpr bool is_valid_edge(uint32_t bits) const noexcept { + return InlineTfa<N>::table[state][bits].state != 0; + } + constexpr uint8_t edits(uint32_t end) const noexcept { + uint32_t leap = (end - index); + return (leap < window_size<N>()) ? InlineTfa<N>::edits[state][leap] : N + 1; + } + // for pretty graphviz dumping; minimal possible edits given perfect input from here on + constexpr uint32_t min_edits() const noexcept { + uint8_t res = N + 1; + for (uint32_t i = 0; i < window_size<N>(); ++i) { + res = std::min(res, InlineTfa<N>::edits[state][i]); + } + return res; + } + // for pretty graphviz dumping; actual edits needed to reach the word end from a valid state + constexpr uint32_t exact_edits(uint32_t end) const noexcept { + assert(valid()); + uint32_t res = end; + for (uint32_t i = 0; i < window_size<N>(); ++i) { + if (uint32_t e = InlineTfa<N>::edits[state][i]; e <= N) { + res = std::min(res, e + diff(index + i, end)); + } + } + return res; + } + // for pretty graphviz dumping; enable using in map and set + bool operator<(const TfaState &rhs) const noexcept { + return std::tie(index, state) < std::tie(rhs.index, rhs.state); + } + // for pretty graphviz dumping; check for redundant edges + bool operator==(const TfaState &rhs) const noexcept { + return std::tie(index, state) == std::tie(rhs.index, rhs.state); + } +}; + +template <uint8_t N> +struct TableMatcher { + using S = TfaState<N>; + using StateType = TfaState<N>; + using StateParamType = TfaState<N>; + using EdgeType = uint32_t; + + const TableDfa<N>::Lookup *lookup; + const uint32_t end; + const bool cased; + + TableMatcher(const TableDfa<N>::Lookup *lookup_in, uint32_t end_in, bool cased_in) + noexcept : lookup(lookup_in), end(end_in), cased(cased_in) {} + + bool is_cased() const noexcept { return cased; } + static constexpr S start() noexcept { return S::start(); } + + uint8_t match_edit_distance(S s) const noexcept { return s.edits(end); } + bool is_match(S s) const noexcept { return s.edits(end) <= N; } + + static constexpr bool can_match(S s) noexcept { return s.valid(); } + static constexpr bool valid_state(S) noexcept { return true; } + + S match_wildcard(S s) const noexcept { return s.next(0); } + S match_input(S s, uint32_t c) const noexcept { + const auto *slice = lookup[s.index].list.data(); + for (size_t i = 0; i < window_size<N>() && slice[i].input != 0; ++i) { + if (slice[i].input == c) { + return s.next(slice[i].match); + } + } + return match_wildcard(s); + } + + bool has_higher_out_edge(S s, uint32_t c) const noexcept { + if (s.is_valid_edge(0)) { + return true; + } + const auto *slice = lookup[s.index].list.data(); + for (size_t i = 0; i < window_size<N>() && slice[i].input > c; ++i) { + if (s.is_valid_edge(slice[i].match)) { + return true; + } + } + return false; + } + + bool has_exact_explicit_out_edge(S s, uint32_t c) const noexcept { + const auto *slice = lookup[s.index].list.data(); + for (size_t i = 0; i < window_size<N>() && slice[i].input >= c; ++i) { + if (slice[i].input == c) { + return s.is_valid_edge(slice[i].match); + } + } + return false; + } + + uint32_t lowest_higher_explicit_out_edge(S s, uint32_t c) const noexcept { + const auto *slice = lookup[s.index].list.data(); + size_t i = window_size<N>(); + while (i-- > 0) { + if (slice[i].input > c && s.is_valid_edge(slice[i].match)) { + return slice[i].input; + } + } + return 0; + } + + uint32_t smallest_explicit_out_edge(S s) const noexcept { + const auto *slice = lookup[s.index].list.data(); + size_t i = window_size<N>(); + while (i-- > 0) { + if (slice[i].input != 0 && s.is_valid_edge(slice[i].match)) { + return slice[i].input; + } + } + return 0; + } + + static constexpr bool valid_edge(uint32_t) noexcept { return true; } + static constexpr uint32_t edge_to_u32char(uint32_t c) noexcept { return c; } + S edge_to_state(S s, uint32_t c) const noexcept { return match_input(s, c); } + + static constexpr bool implies_exact_match_suffix(S) noexcept { return false; } + static constexpr void emit_exact_match_suffix(S, std::vector<uint32_t> &) {} // not called + static constexpr void emit_exact_match_suffix(S, std::string &) {} // not called +}; + +} // unnamed + +template <uint8_t N> +auto +TableDfa<N>::make_lookup(const std::vector<uint32_t> &str)->std::vector<Lookup> +{ + std::vector<Lookup> result(str.size() + 1); + auto have_already = [&](uint32_t c, size_t i)noexcept{ + for (size_t j = 0; j < window_size(); ++j) { + if (result[i].list[j].input == c) { + return true; + } + } + return false; + }; + auto make_vector = [&](uint32_t c, size_t i)noexcept{ + uint32_t bits = 0; + for (size_t j = 0; j < window_size(); ++j) { + bool found = ((i + j) < str.size()) && (str[i+j] == c); + bits = (bits << 1) + found; + } + return bits; + }; + for (size_t i = 0; i < str.size(); ++i) { + for (size_t j = 0; j < window_size(); ++j) { + assert(result[i].list[j].input == 0); + assert(result[i].list[j].match == 0); + if ((i + j) < str.size()) { + uint32_t c = str[i + j]; + if (!have_already(c, i)) { + result[i].list[j].input = c; + result[i].list[j].match = make_vector(c, i); + } + } + } + std::sort(result[i].list.begin(), result[i].list.end(), + [](const auto &a, const auto &b){ return a.input > b.input; }); + } + return result; +} + +template <uint8_t N> +TableDfa<N>::TableDfa(std::vector<uint32_t> str, bool is_cased) + : _lookup(make_lookup(str)), + _is_cased(is_cased) +{ +} + +template <uint8_t N> +TableDfa<N>::~TableDfa() = default; + +template <uint8_t N> +LevenshteinDfa::MatchResult +TableDfa<N>::match(std::string_view u8str) const +{ + TableMatcher<N> matcher(_lookup.data(), _lookup.size() - 1, _is_cased); + return MatchAlgorithm<N>::match(matcher, u8str); +} + +template <uint8_t N> +LevenshteinDfa::MatchResult +TableDfa<N>::match(std::string_view u8str, std::string& successor_out) const +{ + TableMatcher<N> matcher(_lookup.data(), _lookup.size() - 1, _is_cased); + return MatchAlgorithm<N>::match(matcher, u8str, successor_out); +} + +template <uint8_t N> +LevenshteinDfa::MatchResult +TableDfa<N>::match(std::string_view u8str, std::vector<uint32_t>& successor_out) const +{ + TableMatcher<N> matcher(_lookup.data(), _lookup.size() - 1, _is_cased); + return MatchAlgorithm<N>::match(matcher, u8str, successor_out); +} + +template <uint8_t N> +size_t +TableDfa<N>::memory_usage() const noexcept +{ + return _lookup.size() * sizeof(Lookup); +} + +template <uint8_t N> +void +TableDfa<N>::dump_as_graphviz(std::ostream &os) const +{ + using state_t = TfaState<N>; + auto id_of = [state_ids = std::map<state_t,uint32_t>()](state_t state) mutable { + return state_ids.emplace(state, state_ids.size()).first->second; + }; + auto explore = [explored = std::set<state_t>()](state_t state) mutable { + return explored.insert(state).second; + }; + auto edits = [end = (_lookup.size() - 1)](state_t state) noexcept -> uint32_t { + return state.exact_edits(end); + }; + struct Edge { + uint32_t input; + state_t from; + state_t to; + operator bool() const { return to.valid(); } + }; + auto best_edge = [&](const Edge &a, const Edge &b) { + // inverted due to priority queue + if (a.to.min_edits() == b.to.min_edits()) { + return edits(a.to) > edits(b.to); + } + return a.to.min_edits() > b.to.min_edits(); + }; + auto todo = std::priority_queue<Edge,std::vector<Edge>,decltype(best_edge)>(best_edge); + auto handle_state = [&](state_t state) { + if (explore(state)) { + // number states by following the best edges first + auto my_id = id_of(state); + if (edits(state) <= N) { + os << " " << my_id << " [label=\"" << my_id << "(" << edits(state) << ")\", style=\"filled\"];\n"; + } + auto null_edge = Edge{0, state, state.next(0)}; + if (null_edge) { + todo.push(null_edge); + } + for (auto [c, bits]: _lookup[state.index].list) { + if (auto edge = Edge{c, state, state.next(bits)}; edge && edge.to != null_edge.to) { + // only process valid out edges that are not covered by the null_edge + todo.push(edge); + } + } + } + }; + auto handle_edge = [&](Edge edge) { + handle_state(edge.to); + if (edge.input != 0) { + std::string as_utf8; + append_utf32_char(as_utf8, edge.input); + os << " " << id_of(edge.from) << " -> " << id_of(edge.to) << " [label=\"" << as_utf8 << "\"];\n"; + } else { + os << " " << id_of(edge.from) << " -> " << id_of(edge.to) << " [label=\"*\"];\n"; + } + }; + os << std::dec << "digraph table_dfa {\n"; + os << " fontname=\"Helvetica,Arial,sans-serif\"\n"; + os << " node [shape=circle, fontname=\"Helvetica,Arial,sans-serif\", fixedsize=true];\n"; + os << " edge [fontname=\"Helvetica,Arial,sans-serif\"];\n"; + handle_state(state_t::start()); + while (!todo.empty()) { + auto next_edge = todo.top(); + todo.pop(); + handle_edge(next_edge); + } + os << "}\n"; +} + +} diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.h b/vespalib/src/vespa/vespalib/stllike/hashtable.h index e290d2f626c..fa88bb038b4 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.h +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.h @@ -62,7 +62,7 @@ public: class prime_modulator { public: - prime_modulator(next_t sizeOfHashTable) noexcept : _modulo(sizeOfHashTable) { } + explicit prime_modulator(next_t sizeOfHashTable) noexcept : _modulo(sizeOfHashTable) { } next_t modulo(next_t hash) const noexcept { return hash % _modulo; } next_t getTableSize() const noexcept { return _modulo; } static next_t selectHashTableSize(size_t sz) { return hashtable_base::getModuloStl(sz); } @@ -76,7 +76,7 @@ public: class and_modulator { public: - and_modulator(next_t sizeOfHashTable) noexcept : _mask(sizeOfHashTable-1) { } + explicit and_modulator(next_t sizeOfHashTable) noexcept : _mask(sizeOfHashTable-1) { } next_t modulo(next_t hash) const noexcept { return hash & _mask; } next_t getTableSize() const noexcept { return _mask + 1; } static next_t selectHashTableSize(size_t sz) noexcept { return hashtable_base::getModuloSimple(sz); } @@ -198,7 +198,7 @@ public: using pointer = Value*; using iterator_category = std::forward_iterator_tag; - constexpr iterator(hashtable * hash) noexcept : _current(0), _hashTable(hash) { + constexpr explicit iterator(hashtable * hash) noexcept : _current(0), _hashTable(hash) { if (! _hashTable->_nodes[_current].valid()) { advanceToNextValidHash(); } @@ -242,7 +242,7 @@ public: using pointer = const Value*; using iterator_category = std::forward_iterator_tag; - constexpr const_iterator(const hashtable * hash) noexcept : _current(0), _hashTable(hash) { + constexpr explicit const_iterator(const hashtable * hash) noexcept : _current(0), _hashTable(hash) { if (! _hashTable->_nodes[_current].valid()) { advanceToNextValidHash(); } @@ -282,7 +282,7 @@ public: hashtable & operator = (hashtable &&) noexcept = default; hashtable(const hashtable &); hashtable & operator = (const hashtable &); - hashtable(size_t reservedSpace); + explicit hashtable(size_t reservedSpace); hashtable(size_t reservedSpace, const Hash & hasher, const Equal & equal); virtual ~hashtable(); constexpr iterator begin() noexcept { return iterator(this); } diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.hpp b/vespalib/src/vespa/vespalib/stllike/hashtable.hpp index 6d2d397a887..040e421f68c 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.hpp @@ -53,8 +53,7 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::hashtable(size_t rese _nodes(createStore<NodeStore>(reservedSpace, _modulator.getTableSize())), _hasher(hasher), _equal(equal) -{ -} +{ } template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::hashtable(const hashtable &) = default; @@ -130,7 +129,7 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::insert_internal(V && _count++; return insert_result(iterator(this, h), true); } - return insert_internal_cold(std::move(node), h); + return insert_internal_cold(std::forward<V>(node), h); } template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > diff --git a/vespalib/src/vespa/vespalib/text/utf8.h b/vespalib/src/vespa/vespalib/text/utf8.h index 99e3f8cfe13..489b16b1ed4 100644 --- a/vespalib/src/vespa/vespalib/text/utf8.h +++ b/vespalib/src/vespa/vespalib/text/utf8.h @@ -321,6 +321,7 @@ public: return i; } + const char* get_current_ptr() const noexcept { return _p; } }; diff --git a/vespalib/src/vespa/vespalib/util/address_space.h b/vespalib/src/vespa/vespalib/util/address_space.h index 948217bfd2b..fd172427720 100644 --- a/vespalib/src/vespa/vespalib/util/address_space.h +++ b/vespalib/src/vespa/vespalib/util/address_space.h @@ -2,6 +2,7 @@ #pragma once +#include <cstddef> #include <iosfwd> namespace vespalib { diff --git a/vespalog/src/vespa/log/log.h b/vespalog/src/vespa/log/log.h index 857bf4f2b97..edec06dae5e 100644 --- a/vespalog/src/vespa/log/log.h +++ b/vespalog/src/vespa/log/log.h @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <cstdarg> #include <memory> #include <new> // for placement new #include <sys/types.h> // for pid_t diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index 80646dc5607..1bed85b1c02 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -166,19 +166,19 @@ public class Curator extends AbstractComponent implements AutoCloseable { private void addLoggingListener() { curatorFramework.getConnectionStateListenable().addListener((curatorFramework, connectionState) -> { switch (connectionState) { - case SUSPENDED: LOG.info("ZK connection state change: SUSPENDED"); break; - case RECONNECTED: LOG.info("ZK connection state change: RECONNECTED"); break; - case LOST: LOG.warning("ZK connection state change: LOST"); break; + case SUSPENDED -> LOG.info("ZK connection state change: SUSPENDED"); + case RECONNECTED -> LOG.info("ZK connection state change: RECONNECTED"); + case LOST -> LOG.warning("ZK connection state change: LOST"); } }); } - public CompletionWaiter getCompletionWaiter(Path waiterPath, String id, Duration waitForAll) { - return CuratorCompletionWaiter.create(this, waiterPath, id, waitForAll); + public CompletionWaiter getCompletionWaiter(Path barrierPath, String id, Duration waitForAll) { + return CuratorCompletionWaiter.create(this, barrierPath, id, waitForAll); } - public CompletionWaiter createCompletionWaiter(Path waiterPath, String id, Duration waitForAll) { - return CuratorCompletionWaiter.createAndInitialize(this, waiterPath, id, waitForAll); + public CompletionWaiter createCompletionWaiter(Path barrierPath, String id, Duration waitForAll) { + return CuratorCompletionWaiter.createAndInitialize(this, barrierPath, id, waitForAll); } /** Creates a listenable cache which keeps in sync with changes to all the immediate children of a path */ diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java index 7d918baaf54..9a8b9b5bf60 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.curator; import com.yahoo.path.Path; +import com.yahoo.vespa.curator.transaction.CuratorOperations; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; import java.time.Clock; import java.time.Duration; @@ -120,11 +122,13 @@ class CuratorCompletionWaiter implements CompletionWaiter { return new CuratorCompletionWaiter(curator, barrierPath, id, Clock.systemUTC(), waitForAll); } - public static CompletionWaiter createAndInitialize(Curator curator, Path waiterPath, String id, Duration waitForAll) { - curator.delete(waiterPath); - curator.createAtomically(waiterPath); + public static CompletionWaiter createAndInitialize(Curator curator, Path barrierPath, String id, Duration waitForAll) { + // Note: Should be done atomically, but unable to that when path may not exist before delete + // and create should be able to create any missing parent paths + curator.delete(barrierPath); + curator.create(barrierPath); - return new CuratorCompletionWaiter(curator, waiterPath, id, Clock.systemUTC(), waitForAll); + return new CuratorCompletionWaiter(curator, barrierPath, id, Clock.systemUTC(), waitForAll); } private int barrierMemberCount() { diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java index 592b9fc2a05..e1376fb154b 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java @@ -82,12 +82,12 @@ public class MockCurator extends Curator { } @Override - public CompletionWaiter getCompletionWaiter(Path parentPath, String id, Duration waitForAll) { + public CompletionWaiter getCompletionWaiter(Path barrierPath, String id, Duration waitForAll) { return mockFramework().createCompletionWaiter(); } @Override - public CompletionWaiter createCompletionWaiter(Path waiterPath, String id, Duration waitForAll) { + public CompletionWaiter createCompletionWaiter(Path barrierPath, String id, Duration waitForAll) { return mockFramework().createCompletionWaiter(); } |