diff options
61 files changed, 617 insertions, 635 deletions
diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index a8766c9af0b..d25a0fcb8cc 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -789,10 +789,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@9.0.0": - version "9.0.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.0.0.tgz#1a9e4b4c96d8c7886e0110ed310a0135144a1691" - integrity sha512-RThY/MnKrhubF6+s1JflwUjPEsnCEmYCWwqa/aRISKWNXGZ9epUwft4bUMM35SdKF9xvBrLydAM1RDHd1Z//ZQ== +"@eslint/js@9.1.1": + version "9.1.1" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.1.1.tgz#eb0f82461d12779bbafc1b5045cde3143d350a8a" + integrity sha512-5WoDz3Y19Bg2BnErkZTp0en+c/i9PvgFS7MBe1+m60HjFr0hrphlAGp4yzI7pxpt4xShln4ZyYp4neJm8hmOkQ== "@floating-ui/core@^1.4.2": version "1.5.0" @@ -863,10 +863,10 @@ dependencies: prop-types "^15.8.1" -"@humanwhocodes/config-array@^0.12.3": - version "0.12.3" - resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.12.3.tgz#a6216d90f81a30bedd1d4b5d799b47241f318072" - integrity sha512-jsNnTBlMWuTpDkeE3on7+dWJi0D6fdDfeANj/w7MpS8ztROCoLvIO2nG0CcFj+E4k8j4QrSTh4Oryi3i2G669g== +"@humanwhocodes/config-array@^0.13.0": + version "0.13.0" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.13.0.tgz#fb907624df3256d04b9aa2df50d7aa97ec648748" + integrity sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw== dependencies: "@humanwhocodes/object-schema" "^2.0.3" debug "^4.3.1" @@ -882,6 +882,11 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz#4a2868d75d6d6963e423bcf90b7fd1be343409d3" integrity sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA== +"@humanwhocodes/retry@^0.2.3": + version "0.2.3" + resolved "https://registry.yarnpkg.com/@humanwhocodes/retry/-/retry-0.2.3.tgz#c9aa036d1afa643f1250e83150f39efb3a15a631" + integrity sha512-X38nUbachlb01YMlvPFojKoiXq+LzZvuSce70KPMPdeM1Rj03k4dR7lDslhbqXn3Ang4EU3+EAmwEAsbrjHW3g== + "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz#fd3db1d59ecf7cf121e80650bb86712f9b55eced" @@ -1320,85 +1325,85 @@ resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.15.3.tgz#d2509048d69dbb72d5389a14945339f1430b2d3c" integrity sha512-Oy8rmScVrVxWZVOpEF57ovlnhpZ8CCPlnIIumVcV9nFdiSIrus99+Lw78ekXyGvVDlIsFJbSfmSovJUhCWYV3w== -"@rollup/rollup-android-arm-eabi@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.14.3.tgz#bddf05c3387d02fac04b6b86b3a779337edfed75" - integrity sha512-X9alQ3XM6I9IlSlmC8ddAvMSyG1WuHk5oUnXGw+yUBs3BFoTizmG1La/Gr8fVJvDWAq+zlYTZ9DBgrlKRVY06g== - -"@rollup/rollup-android-arm64@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.14.3.tgz#b26bd09de58704c0a45e3375b76796f6eda825e4" - integrity sha512-eQK5JIi+POhFpzk+LnjKIy4Ks+pwJ+NXmPxOCSvOKSNRPONzKuUvWE+P9JxGZVxrtzm6BAYMaL50FFuPe0oWMQ== - -"@rollup/rollup-darwin-arm64@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.14.3.tgz#c5f3fd1aa285b6d33dda6e3f3ca395f8c37fd5ca" - integrity sha512-Od4vE6f6CTT53yM1jgcLqNfItTsLt5zE46fdPaEmeFHvPs5SjZYlLpHrSiHEKR1+HdRfxuzXHjDOIxQyC3ptBA== - -"@rollup/rollup-darwin-x64@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.14.3.tgz#8e4673734d7dc9d68f6d48e81246055cda0e840f" - integrity sha512-0IMAO21axJeNIrvS9lSe/PGthc8ZUS+zC53O0VhF5gMxfmcKAP4ESkKOCwEi6u2asUrt4mQv2rjY8QseIEb1aw== - -"@rollup/rollup-linux-arm-gnueabihf@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.14.3.tgz#53ed38eb13b58ababdb55a7f66f0538a7f85dcba" - integrity sha512-ge2DC7tHRHa3caVEoSbPRJpq7azhG+xYsd6u2MEnJ6XzPSzQsTKyXvh6iWjXRf7Rt9ykIUWHtl0Uz3T6yXPpKw== - -"@rollup/rollup-linux-arm-musleabihf@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.14.3.tgz#0706ee38330e267a5c9326956820f009cfb21fcd" - integrity sha512-ljcuiDI4V3ySuc7eSk4lQ9wU8J8r8KrOUvB2U+TtK0TiW6OFDmJ+DdIjjwZHIw9CNxzbmXY39wwpzYuFDwNXuw== - -"@rollup/rollup-linux-arm64-gnu@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.14.3.tgz#426fce7b8b242ac5abd48a10a5020f5a468c6cb4" - integrity sha512-Eci2us9VTHm1eSyn5/eEpaC7eP/mp5n46gTRB3Aar3BgSvDQGJZuicyq6TsH4HngNBgVqC5sDYxOzTExSU+NjA== - -"@rollup/rollup-linux-arm64-musl@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.14.3.tgz#65bf944530d759b50d7ffd00dfbdf4125a43406f" - integrity sha512-UrBoMLCq4E92/LCqlh+blpqMz5h1tJttPIniwUgOFJyjWI1qrtrDhhpHPuFxULlUmjFHfloWdixtDhSxJt5iKw== - -"@rollup/rollup-linux-powerpc64le-gnu@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.14.3.tgz#494ba3b31095e9a45df9c3f646d21400fb631a95" - integrity sha512-5aRjvsS8q1nWN8AoRfrq5+9IflC3P1leMoy4r2WjXyFqf3qcqsxRCfxtZIV58tCxd+Yv7WELPcO9mY9aeQyAmw== - -"@rollup/rollup-linux-riscv64-gnu@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.14.3.tgz#8b88ed0a40724cce04aa15374ebe5ba4092d679f" - integrity sha512-sk/Qh1j2/RJSX7FhEpJn8n0ndxy/uf0kI/9Zc4b1ELhqULVdTfN6HL31CDaTChiBAOgLcsJ1sgVZjWv8XNEsAQ== - -"@rollup/rollup-linux-s390x-gnu@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.14.3.tgz#09c9e5ec57a0f6ec3551272c860bb9a04b96d70f" - integrity sha512-jOO/PEaDitOmY9TgkxF/TQIjXySQe5KVYB57H/8LRP/ux0ZoO8cSHCX17asMSv3ruwslXW/TLBcxyaUzGRHcqg== - -"@rollup/rollup-linux-x64-gnu@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.14.3.tgz#197f27fd481ad9c861021d5cbbf21793922a631c" - integrity sha512-8ybV4Xjy59xLMyWo3GCfEGqtKV5M5gCSrZlxkPGvEPCGDLNla7v48S662HSGwRd6/2cSneMQWiv+QzcttLrrOA== - -"@rollup/rollup-linux-x64-musl@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.14.3.tgz#5cc0522f4942f2df625e9bfb6fb02c6580ffbce6" - integrity sha512-s+xf1I46trOY10OqAtZ5Rm6lzHre/UiLA1J2uOhCFXWkbZrJRkYBPO6FhvGfHmdtQ3Bx793MNa7LvoWFAm93bg== - -"@rollup/rollup-win32-arm64-msvc@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.14.3.tgz#a648122389d23a7543b261fba082e65fefefe4f6" - integrity sha512-+4h2WrGOYsOumDQ5S2sYNyhVfrue+9tc9XcLWLh+Kw3UOxAvrfOrSMFon60KspcDdytkNDh7K2Vs6eMaYImAZg== - -"@rollup/rollup-win32-ia32-msvc@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.14.3.tgz#34727b5c7953c35fc6e1ae4f770ad3a2025f8e03" - integrity sha512-T1l7y/bCeL/kUwh9OD4PQT4aM7Bq43vX05htPJJ46RTI4r5KNt6qJRzAfNfM+OYMNEVBWQzR2Gyk+FXLZfogGw== - -"@rollup/rollup-win32-x64-msvc@4.14.3": - version "4.14.3" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.14.3.tgz#5b2fb4d8cd44c05deef8a7b0e6deb9ccb8939d18" - integrity sha512-/BypzV0H1y1HzgYpxqRaXGBRqfodgoBBCcsrujT6QRcakDQdfU+Lq9PENPh5jB4I44YWq+0C2eHsHya+nZY1sA== +"@rollup/rollup-android-arm-eabi@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.15.0.tgz#28c9c79c5baccb59a96afcf60e428ea6965a5579" + integrity sha512-O63bJ7p909pRRQfOJ0k/Jp8gNFMud+ZzLLG5EBWquylHxmRT2k18M2ifg8WyjCgFVdpA7+rI0YZ8EkAtg6dSUw== + +"@rollup/rollup-android-arm64@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.15.0.tgz#a2bdafdb753ece571956289a5ba8c37af748bd0c" + integrity sha512-5UywPdmC9jiVOShjQx4uuIcnTQOf85iA4jgg8bkFoH5NYWFfAfrJpv5eeokmTdSmYwUTT5IrcrBCJNkowhrZDA== + +"@rollup/rollup-darwin-arm64@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.15.0.tgz#4cb44cfec3068f6f76f70463ccc25f3e245af06a" + integrity sha512-hNkt75uFfWpRxHItCBmbS0ba70WnibJh6yz60WShSWITLlVRbkvAu1E/c7RlliPY4ajhqJd0UPZz//gNalTd4g== + +"@rollup/rollup-darwin-x64@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.15.0.tgz#1035bfbf53e6acf16771191f41c3d3aff089e8f1" + integrity sha512-HnC5bTP7qdfO9nUw/mBhNcjOEZfbS8NwV+nFegiMhYOn1ATAGZF4kfAxR9BuZevBrebWCxMmxm8NCU1CUoz+wQ== + +"@rollup/rollup-linux-arm-gnueabihf@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.15.0.tgz#0036b835f17ca9e84c188c419399493bd5739986" + integrity sha512-QGOIQIJZeIIqMsc4BUGe8TnV4dkXhSW2EhaQ1G4LqMUNpkyeLztvlDlOoNHn7SR7a4dBANdcEbPkkEzz3rzjzA== + +"@rollup/rollup-linux-arm-musleabihf@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.15.0.tgz#c44420167203400ba7a707f8205413c2817cdaeb" + integrity sha512-PS/Cp8CinYgoysQ8i4UXYH/TZl06fXszvY/RDkyBYgUB1+tKyOMS925/4FZhfrhkl3XQEKjMc3BKtsxpB9Tz9Q== + +"@rollup/rollup-linux-arm64-gnu@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.15.0.tgz#531d3792e1526583ecd794ceee0ab980d79813dd" + integrity sha512-XzOsnD6lGDP+k+vGgTYAryVGu8N89qpjMN5BVFUj75dGVFP3FzIVAufJAraxirpDwEQZA7Gjs0Vo5p4UmnnjsA== + +"@rollup/rollup-linux-arm64-musl@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.15.0.tgz#86376eaa6d65a860a046e0dfe285a51792bc2026" + integrity sha512-+ScJA4Epbx/ZQGjDnbvTAcb8ZD06b+TlIka2UkujbKf1I/A+yrvEcJwG3/27zMmvcWMQyeCJhbL9TlSjzL0B7Q== + +"@rollup/rollup-linux-powerpc64le-gnu@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.15.0.tgz#6adf69ce27d1266dbb86eeac237ad5dd4d4a1f28" + integrity sha512-1cUSvYgnyTakM4FDyf/GxUCDcqmj/hUh1NOizEOJU7+D5xEfFGCxgcNOs3hYBeRMUCcGmGkt01EhD3ILgKpGHQ== + +"@rollup/rollup-linux-riscv64-gnu@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.15.0.tgz#961c290372d170f588ebf65c145c485c4aad7005" + integrity sha512-3A1FbHDbBUvpJXFAZwVsiROIcstVHP9AX/cwnyIhAp+xyQ1cBCxywKtuzmw0Av1MDNNg/y/9dDHtNypfRa8bdw== + +"@rollup/rollup-linux-s390x-gnu@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.15.0.tgz#cb43301e10f17f0a642416c5d3d82a26cf430fa8" + integrity sha512-hYPbhg9ow6/mXIkojc8LOeiip2sCTuw1taWyoOXTOWk9vawIXz8x7B4KkgWUAtvAElssxhSyEXr2EZycH/FGzQ== + +"@rollup/rollup-linux-x64-gnu@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.15.0.tgz#25c1fb87b2255949ee7ff6956e205710c5d7c414" + integrity sha512-511qln5mPSUKwv7HI28S1jCD1FK+2WbX5THM9A9annr3c1kzmfnf8Oe3ZakubEjob3IV6OPnNNcesfy+adIrmw== + +"@rollup/rollup-linux-x64-musl@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.15.0.tgz#145745e339e282c7afc36142bd5a3f9495c7c681" + integrity sha512-4qKKGTDIv2bQZ+afhPWqPL+94+dLtk4lw1iwbcylKlLNqQ/Yyjof2CFYBxf6npiDzPV+zf4EWRiHb26/4Vsm9w== + +"@rollup/rollup-win32-arm64-msvc@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.15.0.tgz#95dae687b645a25aab3a082d987556f58274ffbe" + integrity sha512-nEtaFBHp1OnbOf+tz66DtID579sNRHGgMC23to8HUyVuOCpCMD0CvRNqiDGLErLNnwApWIUtUl1VvuovCWUxwg== + +"@rollup/rollup-win32-ia32-msvc@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.15.0.tgz#acbd48f10093e6cd52f99ad004966433a49cb362" + integrity sha512-5O49NykwSgX6iT2HgZ6cAoGHt6T/FqNMB5OqFOGxU/y1GyFSHquox1sK2OqApQc0ANxiHFQEMNDLNVCL7AUDnQ== + +"@rollup/rollup-win32-x64-msvc@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.15.0.tgz#68bb584231dfc8e36bb7ad5317dfd1fd2563a6f7" + integrity sha512-YA0hTwCunmKNeTOFWdJuKhdXse9jBqgo34FDo+9aS0spfCkp+wj0o1bCcOOTu+0P48O95GTfkLTAaVonwNuIdQ== "@sinclair/typebox@^0.27.8": version "0.27.8" @@ -2664,16 +2669,17 @@ eslint-visitor-keys@^4.0.0: integrity sha512-OtIRv/2GyiF6o/d8K7MYKKbXrOUBIK6SfkIRM4Z0dY3w+LiQ0vy3F57m0Z71bjbyeiWFiHJ8brqnmE6H6/jEuw== eslint@^9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.0.0.tgz#6270548758e390343f78c8afd030566d86927d40" - integrity sha512-IMryZ5SudxzQvuod6rUdIUz29qFItWx281VhtFVc2Psy/ZhlCeD/5DT6lBIJ4H3G+iamGJoTln1v+QSuPw0p7Q== + version "9.1.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.1.0.tgz#262625f6b0921f7550f128a0098d05ecaad989c6" + integrity sha512-1TCBecGFQtItia2o39P7Z4BK1X7ByNPxAiWJvwiyTGcOwYnTiiASgMpNA6a+beu8cFPhEDWvPf6mIlYUJv6sgA== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.6.1" "@eslint/eslintrc" "^3.0.2" - "@eslint/js" "9.0.0" - "@humanwhocodes/config-array" "^0.12.3" + "@eslint/js" "9.1.1" + "@humanwhocodes/config-array" "^0.13.0" "@humanwhocodes/module-importer" "^1.0.1" + "@humanwhocodes/retry" "^0.2.3" "@nodelib/fs.walk" "^1.2.8" ajv "^6.12.4" chalk "^4.0.0" @@ -2689,7 +2695,6 @@ eslint@^9.0.0: file-entry-cache "^8.0.0" find-up "^5.0.0" glob-parent "^6.0.2" - graphemer "^1.4.0" ignore "^5.2.0" imurmurhash "^0.1.4" is-glob "^4.0.0" @@ -3078,11 +3083,6 @@ graceful-fs@^4.2.4, graceful-fs@^4.2.9: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== -graphemer@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" - integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== - has-bigints@^1.0.1, has-bigints@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa" @@ -4884,28 +4884,28 @@ reusify@^1.0.4: integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== rollup@^4.13.0: - version "4.14.3" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.14.3.tgz#bcbb7784b35826d3164346fa6d5aac95190d8ba9" - integrity sha512-ag5tTQKYsj1bhrFC9+OEWqb5O6VYgtQDO9hPDBMmIbePwhfSr+ExlcU741t8Dhw5DkPCQf6noz0jb36D6W9/hw== + version "4.15.0" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.15.0.tgz#3be428e4fe86297b1b3448f29515d978593d9d9a" + integrity sha512-i0ir57IMF5o7YvNYyUNeIGG+IZaaucnGZAOsSctO2tPLXlCEaZzyBa+QhpHNSgtpyLMoDev2DyN6a7J1dQA8Tw== dependencies: "@types/estree" "1.0.5" optionalDependencies: - "@rollup/rollup-android-arm-eabi" "4.14.3" - "@rollup/rollup-android-arm64" "4.14.3" - "@rollup/rollup-darwin-arm64" "4.14.3" - "@rollup/rollup-darwin-x64" "4.14.3" - "@rollup/rollup-linux-arm-gnueabihf" "4.14.3" - "@rollup/rollup-linux-arm-musleabihf" "4.14.3" - "@rollup/rollup-linux-arm64-gnu" "4.14.3" - "@rollup/rollup-linux-arm64-musl" "4.14.3" - "@rollup/rollup-linux-powerpc64le-gnu" "4.14.3" - "@rollup/rollup-linux-riscv64-gnu" "4.14.3" - "@rollup/rollup-linux-s390x-gnu" "4.14.3" - "@rollup/rollup-linux-x64-gnu" "4.14.3" - "@rollup/rollup-linux-x64-musl" "4.14.3" - "@rollup/rollup-win32-arm64-msvc" "4.14.3" - "@rollup/rollup-win32-ia32-msvc" "4.14.3" - "@rollup/rollup-win32-x64-msvc" "4.14.3" + "@rollup/rollup-android-arm-eabi" "4.15.0" + "@rollup/rollup-android-arm64" "4.15.0" + "@rollup/rollup-darwin-arm64" "4.15.0" + "@rollup/rollup-darwin-x64" "4.15.0" + "@rollup/rollup-linux-arm-gnueabihf" "4.15.0" + "@rollup/rollup-linux-arm-musleabihf" "4.15.0" + "@rollup/rollup-linux-arm64-gnu" "4.15.0" + "@rollup/rollup-linux-arm64-musl" "4.15.0" + "@rollup/rollup-linux-powerpc64le-gnu" "4.15.0" + "@rollup/rollup-linux-riscv64-gnu" "4.15.0" + "@rollup/rollup-linux-s390x-gnu" "4.15.0" + "@rollup/rollup-linux-x64-gnu" "4.15.0" + "@rollup/rollup-linux-x64-musl" "4.15.0" + "@rollup/rollup-win32-arm64-msvc" "4.15.0" + "@rollup/rollup-win32-ia32-msvc" "4.15.0" + "@rollup/rollup-win32-x64-msvc" "4.15.0" fsevents "~2.3.2" rsvp@^4.8.4: @@ -5521,9 +5521,9 @@ v8-to-istanbul@^9.0.1: convert-source-map "^1.6.0" vite@^5.0.5: - version "5.2.9" - resolved "https://registry.yarnpkg.com/vite/-/vite-5.2.9.tgz#cd9a356c6ff5f7456c09c5ce74068ffa8df743d9" - integrity sha512-uOQWfuZBlc6Y3W/DTuQ1Sr+oIXWvqljLvS881SVmAj00d5RdgShLcuXWxseWPd4HXwiYBFW/vXHfKFeqj9uQnw== + version "5.2.10" + resolved "https://registry.yarnpkg.com/vite/-/vite-5.2.10.tgz#2ac927c91e99d51b376a5c73c0e4b059705f5bd7" + integrity sha512-PAzgUZbP7msvQvqdSD+ErD5qGnSFiGOoWmV5yAKUEI0kdhjbH6nMWVyZQC/hSc4aXwc0oJ9aEdIiF9Oje0JFCw== dependencies: esbuild "^0.20.1" postcss "^8.4.38" diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index 053a5ac777b..7659a1e6562 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -159,7 +159,7 @@ public class ConvertParsedFields { var dataType = field.getDataType(); var otherType = summaryField.getType(); if (otherType != null && summaryField.getHasExplicitType()) { - schema.getDeployLogger().log(Level.FINE, () -> "For schema '" + schema.getName() + + schema.getDeployLogger().log(Level.WARNING, () -> "For schema '" + schema.getName() + "', field '" + field.getName() + "', summary '" + summaryField.name() + "': Specifying the type is deprecated, ignored and will be an error in Vespa 9." + diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java index ee15b95b198..3c87044850f 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -233,7 +233,7 @@ public class ConvertParsedSchemas { var parsedType = parsedField.getType(); if (parsedType != null) { var log = schema.getDeployLogger(); - log.log(Level.FINE, () -> "For schema '" + schema.getName() + + log.log(Level.WARNING, () -> "For schema '" + schema.getName() + "', document-summary '" + parsed.name() + "', summary field '" + parsedField.name() + "': Specifying the type is deprecated, ignored and will be an error in Vespa 9." + diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java index 67297245ff1..587ffcb86c7 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java @@ -6,16 +6,12 @@ import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.Field; import com.yahoo.schema.Schema; import com.yahoo.schema.document.Attribute; -import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; -import java.util.LinkedHashSet; -import java.util.Set; - /** * This processor creates a {@link com.yahoo.schema.document.SDDocumentType} for each {@link Schema} * object which holds all the data that search @@ -24,8 +20,6 @@ import java.util.Set; */ public class AddExtraFieldsToDocument extends Processor { - Set<String> extraSummaryFields = new LinkedHashSet<String>(); - AddExtraFieldsToDocument(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } @@ -39,31 +33,7 @@ public class AddExtraFieldsToDocument extends Processor { } for (var docsum : schema.getSummaries().values()) { for (var summaryField : docsum.getSummaryFields().values()) { - var transform = summaryField.getTransform(); - if (transform.isDynamic() && DynamicSummaryTransformUtils.summaryFieldIsRequiredInDocumentType(summaryField) || - transform == SummaryTransform.NONE || - transform == SummaryTransform.DOCUMENT_ID) - { - // TODO: Adding the 'documentid' field should no longer be needed when the docsum framework in the backend has been simplified and the transform is always used. - addSummaryField(schema, document, summaryField, validate); - } else { - // skip: generated from attribute or similar, - // so does not need to be included as an extra - // field in the document type - } - } - } - /* - * Don't use extra summary fields when generating summaries. They will not be created nor populated by - * future vespa versions. When vespa version 'X' stops using these fields and vespa version 'Y' stops - * populating the fields, rollback from vespa version >= 'Y' to vespa version < 'X' will break (missing - * summary fields). - */ - for (var docsum : schema.getSummaries().values()) { - for (var summaryField : docsum.getSummaryFields().values()) { - if (extraSummaryFields.contains(summaryField.getName())) { - considerCopyTransformForExtraSummaryField(schema, summaryField); - } + considerCopyTransformForExtraSummaryField(schema, summaryField); } } } @@ -81,24 +51,6 @@ public class AddExtraFieldsToDocument extends Processor { addField(schema, document, field, validate); } - private void addSummaryField(Schema schema, SDDocumentType document, SummaryField field, boolean validate) { - Field docField = document.getField(field.getName()); - if (docField == null) { - ImmutableSDField existingField = schema.getField(field.getName()); - if (existingField == null) { - SDField newField = new SDField(document, field.getName(), field.getDataType()); - newField.setIsExtraField(true); - extraSummaryFields.add(field.getName()); - document.addField(newField); - } else if (!existingField.isImportedField()) { - document.addField(existingField.asField()); - } - } else if (!docField.getDataType().equals(field.getDataType())) { - if (validate) - throw newProcessException(schema, field, "Summary field has conflicting type."); - } - } - private void addField(Schema schema, SDDocumentType document, Field field, boolean validate) { if (document.getField(field.getName()) != null && !(document.getField(field.getName()) == field)) { if (validate) @@ -116,6 +68,6 @@ public class AddExtraFieldsToDocument extends Processor { private boolean fieldIsExtraSummaryField(Schema schema, String name) { var field = schema.getConcreteField(name); - return field == null || extraSummaryFields.contains(name); + return field == null; } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java b/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java index 2e9c23dbf06..487aff81c68 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java @@ -39,20 +39,6 @@ public class DynamicSummaryTransformUtils { return (type.equals(DataType.getArray(DataType.STRING))); } - /** - * Whether a summary field must be populated by the source field with the given type in an indexing script. - */ - public static boolean summaryFieldIsPopulatedBySourceField(DataType sourceFieldType) { - return false; - } - - /** - * Whether a summary field is required as an extra field in the document type. - */ - public static boolean summaryFieldIsRequiredInDocumentType(SummaryField summaryField) { - return summaryFieldIsPopulatedBySourceField(summaryField.getDataType()); - } - public static String getSource(SummaryField summaryField, Schema schema) { // Summary fields with the original supported type is always present in the document type. // However, if the source of that summary field is a single explicit source that exists in the schema we diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java index b5d1cf71809..51fb6b2b065 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java @@ -34,33 +34,25 @@ public class IndexingOutputs extends Processor { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; - Set<String> summaryFields = new TreeSet<>(); - findSummaryTo(schema, field, summaryFields, summaryFields); - MyConverter converter = new MyConverter(schema, field, summaryFields, validate); + findSummaryTo(schema, field); + MyConverter converter = new MyConverter(schema, field, validate); field.setIndexingScript(schema.getName(), (ScriptExpression)converter.convert(script)); } } - public void findSummaryTo(Schema schema, SDField field, Set<String> dynamicSummary, Set<String> staticSummary) { + public void findSummaryTo(Schema schema, SDField field) { var summaryFields = schema.getSummaryFields(field); - if (summaryFields.isEmpty()) { - fillSummaryToFromField(field, dynamicSummary, staticSummary); - } else { - fillSummaryToFromSearch(schema, field, summaryFields, dynamicSummary, staticSummary); - } + fillSummaryToFromSearch(schema, field, summaryFields); } - private void fillSummaryToFromSearch(Schema schema, SDField field, List<SummaryField> summaryFields, - Set<String> dynamicSummary, Set<String> staticSummary) { + private void fillSummaryToFromSearch(Schema schema, SDField field, List<SummaryField> summaryFields) { for (SummaryField summaryField : summaryFields) { - fillSummaryToFromSummaryField(schema, field, summaryField, dynamicSummary, staticSummary); + fillSummaryToFromSummaryField(schema, field, summaryField); } } - private void fillSummaryToFromSummaryField(Schema schema, SDField field, SummaryField summaryField, - Set<String> dynamicSummary, Set<String> staticSummary) { + private void fillSummaryToFromSummaryField(Schema schema, SDField field, SummaryField summaryField) { SummaryTransform summaryTransform = summaryField.getTransform(); - String summaryName = summaryField.getName(); if (summaryTransform.isDynamic() && summaryField.getSourceCount() > 2) { // Avoid writing to summary fields that have more than a single input field, as that is handled by the // summary rewriter in the search core. @@ -68,30 +60,11 @@ public class IndexingOutputs extends Processor { } if (summaryTransform.isDynamic()) { DataType fieldType = field.getDataType(); - if (!DynamicSummaryTransformUtils.summaryFieldIsPopulatedBySourceField(fieldType)) { - if (!DynamicSummaryTransformUtils.isSupportedType(fieldType)) { - warn(schema, field, "Dynamic summaries are only supported for fields of type " + - "string and array<string>, ignoring summary field '" + summaryField.getName() + - "' for sd field '" + field.getName() + "' of type " + - fieldType.getName() + "."); - } - return; - } - dynamicSummary.add(summaryName); - } else if (summaryTransform != SummaryTransform.ATTRIBUTE && - summaryTransform != SummaryTransform.TOKENS && - summaryTransform != SummaryTransform.ATTRIBUTE_TOKENS) { - staticSummary.add(summaryName); - } - } - - private static void fillSummaryToFromField(SDField field, Set<String> dynamicSummary, Set<String> staticSummary) { - for (SummaryField summaryField : field.getSummaryFields().values()) { - String summaryName = summaryField.getName(); - if (summaryField.getTransform().isDynamic()) { - dynamicSummary.add(summaryName); - } else { - staticSummary.add(summaryName); + if (!DynamicSummaryTransformUtils.isSupportedType(fieldType)) { + warn(schema, field, "Dynamic summaries are only supported for fields of type " + + "string and array<string>, ignoring summary field '" + summaryField.getName() + + "' for sd field '" + field.getName() + "' of type " + + fieldType.getName() + "."); } } } @@ -100,13 +73,11 @@ public class IndexingOutputs extends Processor { final Schema schema; final Field field; - final Set<String> summaryFields; final boolean validate; - MyConverter(Schema schema, Field field, Set<String> summaryFields, boolean validate) { + MyConverter(Schema schema, Field field, boolean validate) { this.schema = schema; this.field = field; - this.summaryFields = summaryFields.isEmpty() ? Set.of(field.getName()) : summaryFields; this.validate = validate; } @@ -134,18 +105,7 @@ public class IndexingOutputs extends Processor { } else if (exp instanceof IndexExpression) { ret.add(new IndexExpression(field.getName())); } else if (exp instanceof SummaryExpression) { - for (String fieldName : summaryFields) { - ret.add(new SummaryExpression(fieldName)); - } - /* - * Write to summary field source. AddExtraFieldsToDocument processor adds the "copy" - * summary transform to summary fields without a corresponding explicitly declared - * document field (2023-11-01). Future vespa versions will stop adding document - * fields for those summary fields. - */ - if (!summaryFields.contains(field.getName())) { - ret.add(new SummaryExpression(field.getName())); - } + ret.add(new SummaryExpression(field.getName())); } else { throw new UnsupportedOperationException(exp.getClass().getName()); } diff --git a/config-model/src/test/derived/imported_struct_fields/index-info.cfg b/config-model/src/test/derived/imported_struct_fields/index-info.cfg index f023328380c..2b8a6fc344d 100644 --- a/config-model/src/test/derived/imported_struct_fields/index-info.cfg +++ b/config-model/src/test/derived/imported_struct_fields/index-info.cfg @@ -9,10 +9,6 @@ indexinfo[].command[].indexname "parent_ref" indexinfo[].command[].command "type Reference<parent>" indexinfo[].command[].indexname "parent_ref" indexinfo[].command[].command "word" -indexinfo[].command[].indexname "documentid" -indexinfo[].command[].command "string" -indexinfo[].command[].indexname "documentid" -indexinfo[].command[].command "type string" indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "my_elem_array.name" diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg index 14a7a62f4bb..0cdf921de25 100644 --- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg +++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg @@ -17,7 +17,7 @@ ilscript[].content[] "clear_state | guard { input loc | to_pos | zcurve | attrib ilscript[].content[] "clear_state | guard { input a | summary a | attribute a; }" ilscript[].content[] "clear_state | guard { input adynamic | summary adynamic | attribute adynamic; }" ilscript[].content[] "clear_state | guard { input abolded | summary abolded | attribute abolded; }" -ilscript[].content[] "clear_state | guard { input b | summary anotherb | summary b; }" +ilscript[].content[] "clear_state | guard { input b | summary b; }" ilscript[].content[] "clear_state | guard { input c | summary c | attribute c; }" ilscript[].content[] "clear_state | guard { input d | summary d; }" ilscript[].content[] "clear_state | guard { input e | summary e; }" diff --git a/config-model/src/test/derived/multiplesummaries/index-info.cfg b/config-model/src/test/derived/multiplesummaries/index-info.cfg index 65ffd71d1ca..58759be9398 100644 --- a/config-model/src/test/derived/multiplesummaries/index-info.cfg +++ b/config-model/src/test/derived/multiplesummaries/index-info.cfg @@ -83,16 +83,6 @@ indexinfo[].command[].indexname "mytags" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mytags" indexinfo[].command[].command "type Array<string>" -indexinfo[].command[].indexname "alltags" -indexinfo[].command[].command "multivalue" -indexinfo[].command[].indexname "alltags" -indexinfo[].command[].command "string" -indexinfo[].command[].indexname "alltags" -indexinfo[].command[].command "type Array<string>" -indexinfo[].command[].indexname "anotherb" -indexinfo[].command[].command "string" -indexinfo[].command[].indexname "anotherb" -indexinfo[].command[].command "type string" indexinfo[].command[].indexname "loc_pos.x" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "loc_pos.x" diff --git a/config-model/src/test/derived/streamingstruct/documentmanager.cfg b/config-model/src/test/derived/streamingstruct/documentmanager.cfg index f916cc26c36..0b503b04926 100644 --- a/config-model/src/test/derived/streamingstruct/documentmanager.cfg +++ b/config-model/src/test/derived/streamingstruct/documentmanager.cfg @@ -143,10 +143,4 @@ doctype[].structtype[].field[].type 10017 doctype[].structtype[].field[].name "g" doctype[].structtype[].field[].internalid 1091070635 doctype[].structtype[].field[].type 10012 -doctype[].structtype[].field[].name "snippet2" -doctype[].structtype[].field[].internalid 1812076817 -doctype[].structtype[].field[].type 10012 -doctype[].structtype[].field[].name "anothersummaryfield" -doctype[].structtype[].field[].internalid 1811005492 -doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java index a11e743b4d2..6252fe62a1d 100644 --- a/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java @@ -40,7 +40,7 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { SDDocumentType document = schema.getDocument(); assertEquals("simple", document.getName()); - assertEquals(20, document.getFieldCount()); + assertEquals(19, document.getFieldCount()); SDField field; Attribute attribute; diff --git a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java index 6442edd547d..8ffbab84fd7 100644 --- a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java @@ -359,7 +359,7 @@ public class SummaryTestCase { ApplicationBuilder.createFromStrings(logger, sd); if (explicit) { assertEquals(1, logger.entries.size()); - assertEquals(Level.FINE, logger.entries.get(0).level); + assertEquals(Level.WARNING, logger.entries.get(0).level); assertEquals("For schema 'test', field 'foo', summary 'bar':" + " Specifying the type is deprecated, ignored and will be an error in Vespa 9." + " Remove the type specification to silence this warning.", logger.entries.get(0).message); @@ -392,7 +392,7 @@ public class SummaryTestCase { ApplicationBuilder.createFromStrings(logger, sd); if (explicit) { assertEquals(1, logger.entries.size()); - assertEquals(Level.FINE, logger.entries.get(0).level); + assertEquals(Level.WARNING, logger.entries.get(0).level); assertEquals("For schema 'test', document-summary 'bar', summary field 'foo':" + " Specifying the type is deprecated, ignored and will be an error in Vespa 9." + " Remove the type specification to silence this warning.", logger.entries.get(0).message); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AddExtraFieldsToDocumentTest.java b/config-model/src/test/java/com/yahoo/schema/processing/AddExtraFieldsToDocumentTest.java index aad6df62993..27e9aae04b5 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/AddExtraFieldsToDocumentTest.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/AddExtraFieldsToDocumentTest.java @@ -43,9 +43,9 @@ public class AddExtraFieldsToDocumentTest { assertSummary(schema, "foo", "my_a", SummaryTransform.COPY, "a"); assertSummary(schema, "foo", "my_b", SummaryTransform.COPY, "b"); assertSummary(schema, "foo", "my_c", SummaryTransform.ATTRIBUTE, "c"); - // Extra fields should still be created - assertField(schema, "my_a", DataType.STRING); - assertField(schema,"my_b", DataType.INT); + // Extra fields should not be created + assertNull(schema.getDocument().getField("my_a")); + assertNull(schema.getDocument().getField("my_b")); assertNull(schema.getDocument().getField("my_c")); } @@ -78,11 +78,4 @@ public class AddExtraFieldsToDocumentTest { assertEquals(transform, field.getTransform()); assertEquals(source, field.getSingleSource()); } - - private void assertField(Schema schema, String name, DataType type) { - var field = schema.getDocument().getField(name); - assertNotNull(field); - assertEquals(field.getDataType(), type); - } - } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java index ff7e43b2936..b9685d9a4ff 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java @@ -37,11 +37,11 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { SDDocumentType docType = schema.getDocument(); assertNotNull(docType); assertNotNull(docType.getField("foo")); - assertNotNull(docType.getField("bar")); - assertNotNull(docType.getField("cox")); + assertNull(docType.getField("bar")); + assertNull(docType.getField("cox")); assertNotNull(docType.getField("mytags")); - assertNotNull(docType.getField("alltags")); - assertEquals(5, docType.getFieldCount()); + assertNull(docType.getField("alltags")); + assertEquals(2, docType.getFieldCount()); } @Test diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java index 135a9fa295a..e4d1b5da29e 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java @@ -34,9 +34,9 @@ public class ImplicitStructTypesTestCase extends AbstractSchemaTestCase { assertNotNull(docType); assertField(docType, "doc_str", DataType.STRING); - assertField(docType, "doc_str_sum", DataType.STRING); + assertNoField(docType, "doc_str_sum"); assertField(docType, "doc_uri", DataType.URI); - assertField(docType, "docsum_str", DataType.STRING); + assertNoField(docType, "docsum_str"); } @SuppressWarnings({ "UnusedDeclaration" }) @@ -60,6 +60,11 @@ public class ImplicitStructTypesTestCase extends AbstractSchemaTestCase { assertTrue(field instanceof SDField); } + private static void assertNoField(SDDocumentType docType, String fieldName) { + var field = getSecretField(docType, fieldName); + assertNull(field); + } + private static Field getSecretField(SDDocumentType docType, String fieldName) { for (Field field : docType.fieldSet()) { if (field.getName().equals(fieldName)) { diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java index f56d2b21a2d..d5af996bd59 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java @@ -78,7 +78,7 @@ public class IndexingOutputsTestCase { """; var builder = ApplicationBuilder.createFromString(sd); var schema = builder.getSchema(); - assertEquals("{ input foo | summary baz | summary bar; }", + assertEquals("{ input foo | summary bar; }", schema.getConcreteField("bar").getIndexingScript().toString()); } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java index 310706cb0d1..de99d46b9ca 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java @@ -64,7 +64,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { field.addSummaryField(createStaticSummaryField(field, "test")); field.addSummaryField(createStaticSummaryField(field, "other")); field.addSummaryField(createDynamicSummaryField(field, "dyn2")); - assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary other | " + + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | " + "summary test | index test; }", field); } @@ -113,7 +113,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { "clear_state | guard { input chatter | tokenize normalize stem:\"BEST\" | index chatter; }", "clear_state | guard { input description | tokenize normalize stem:\"BEST\" | summary description | index description; }", "clear_state | guard { input exactemento_src | lowercase | tokenize normalize stem:\"BEST\" | index exactemento | summary exactemento; }", - "clear_state | guard { input longdesc | summary longdesc | summary longstat; }", + "clear_state | guard { input longdesc | summary longdesc; }", "clear_state | guard { input measurement | attribute measurement | summary measurement; }", "clear_state | guard { input measurement | to_array | attribute measurement_arr; }", "clear_state | guard { input popularity | attribute popularity; }", diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index 72184c5ea32..4d9111b2711 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -9,8 +9,6 @@ import com.yahoo.component.chain.Chain; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.errorhandling.Results; -import com.yahoo.errorhandling.Results.Builder; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; @@ -19,12 +17,12 @@ import com.yahoo.search.Searcher; import com.yahoo.search.federation.selection.FederationTarget; import com.yahoo.search.federation.selection.TargetSelector; import com.yahoo.search.federation.sourceref.ModifyQueryAndResult; +import com.yahoo.search.federation.sourceref.ResolveResult; import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec; import com.yahoo.search.federation.sourceref.SearchChainResolver; import com.yahoo.search.federation.sourceref.SingleTarget; import com.yahoo.search.federation.sourceref.SourceRefResolver; import com.yahoo.search.federation.sourceref.SourcesTarget; -import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; import com.yahoo.search.federation.sourceref.VirtualSourceResolver; import com.yahoo.search.query.Properties; import com.yahoo.search.result.ErrorMessage; @@ -53,6 +51,7 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -206,14 +205,44 @@ public class FederationSearcher extends ForkingSearcher { setRequestTimeoutInMilliseconds(searchChain.requestTimeoutMillis()); } + private static List<String> extractErrors(List<ResolveResult> results) { + List<String> errors = List.of(); + for (ResolveResult result : results) { + if (result.errorMsg() != null) { + if (errors.isEmpty()) { + errors = new ArrayList<>(); + } + errors.add(result.errorMsg()); + } + } + return errors; + } + + private static List<SearchChainInvocationSpec> extractSpecs(List<ResolveResult> results) { + List<SearchChainInvocationSpec> errors = List.of(); + for (ResolveResult result : results) { + if (result.invocationSpec() != null) { + if (errors.isEmpty()) { + errors = List.of(result.invocationSpec()); + } else if (errors.size() == 1) { + errors = new ArrayList<>(errors); + errors.add(result.invocationSpec()); + } else { + errors.add(result.invocationSpec()); + } + } + } + return errors; + } + @Override public Result search(Query query, Execution execution) { Result mergedResults = execution.search(query); var targets = getTargets(query.getModel().getSources(), query.properties()); - warnIfUnresolvedSearchChains(targets.errors(), mergedResults.hits()); + warnIfUnresolvedSearchChains(extractErrors(targets), mergedResults.hits()); - var prunedTargets = pruneTargetsWithoutDocumentTypes(query.getModel().getRestrict(), targets.data()); + var prunedTargets = pruneTargetsWithoutDocumentTypes(query.getModel().getRestrict(), extractSpecs(targets)); var regularTargetHandlers = resolveSearchChains(prunedTargets, execution.searchChainRegistry()); query.errors().addAll(regularTargetHandlers.errors()); @@ -311,32 +340,19 @@ public class FederationSearcher extends ForkingSearcher { .forEach((k, v) -> outgoing.properties().set(k, v)); } - private ErrorMessage missingSearchChainsErrorMessage(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) { - String message = String.join(" ", getMessagesSet(unresolvedSearchChainExceptions)) + + private ErrorMessage missingSearchChainsErrorMessage(List<String> errors) { + String message = String.join(" ", new TreeSet<>(errors)) + " Valid source refs are " + String.join(", ", allSourceRefDescriptions()) +'.'; return ErrorMessage.createInvalidQueryParameter(message); } private List<String> allSourceRefDescriptions() { - List<String> descriptions = new ArrayList<>(); - - for (com.yahoo.search.federation.sourceref.Target target : searchChainResolver.allTopLevelTargets()) - descriptions.add(target.searchRefDescription()); - return descriptions; - } - - private static Set<String> getMessagesSet(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) { - Set<String> messages = new LinkedHashSet<>(); - for (UnresolvedSearchChainException exception : unresolvedSearchChainExceptions) { - messages.add(exception.getMessage()); - } - return messages; + return searchChainResolver.allTopLevelTargets().stream().map(com.yahoo.search.federation.sourceref.Target::searchRefDescription).toList(); } - private void warnIfUnresolvedSearchChains(List<UnresolvedSearchChainException> missingTargets, - HitGroup errorHitGroup) { - if (!missingTargets.isEmpty()) { - errorHitGroup.addError(missingSearchChainsErrorMessage(missingTargets)); + private void warnIfUnresolvedSearchChains(List<String> errorMessages, HitGroup errorHitGroup) { + if (!errorMessages.isEmpty()) { + errorHitGroup.addError(missingSearchChainsErrorMessage(errorMessages)); } } @@ -344,7 +360,7 @@ public class FederationSearcher extends ForkingSearcher { public Collection<CommentedSearchChain> getSearchChainsForwarded(SearchChainRegistry registry) { List<CommentedSearchChain> searchChains = new ArrayList<>(); - for (com.yahoo.search.federation.sourceref.Target target : searchChainResolver.allTopLevelTargets()) { + for (var target : searchChainResolver.allTopLevelTargets()) { if (target instanceof SourcesTarget) { searchChains.addAll(commentedSourceProviderSearchChains((SourcesTarget)target, registry)); } else if (target instanceof SingleTarget) { @@ -468,40 +484,32 @@ public class FederationSearcher extends ForkingSearcher { return orderer; } - private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> getTargets(Set<String> sources, Properties properties) { + private List<ResolveResult> getTargets(Set<String> sources, Properties properties) { return sources.isEmpty() ? defaultSearchChains(properties): resolveSources(sources, properties); } - private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> resolveSources(Set<String> sourcesInQuery, Properties properties) { - Results.Builder<SearchChainInvocationSpec, UnresolvedSearchChainException> result = new Builder<>(); + private List<ResolveResult> resolveSources(Set<String> sourcesInQuery, Properties properties) { + List<ResolveResult> result = new ArrayList<>(); Set<String> sources = virtualSourceResolver.resolve(sourcesInQuery); for (String source : sources) { - try { - result.addAllData(sourceRefResolver.resolve(asSourceSpec(source), properties)); - } catch (UnresolvedSearchChainException e) { - result.addError(e); - } + result.addAll(sourceRefResolver.resolve(asSourceSpec(source), properties)); } - return result.build(); + return List.copyOf(result); } - public Results<SearchChainInvocationSpec, UnresolvedSearchChainException> defaultSearchChains(Properties sourceToProviderMap) { - Results.Builder<SearchChainInvocationSpec, UnresolvedSearchChainException> result = new Builder<>(); + public List<ResolveResult> defaultSearchChains(Properties sourceToProviderMap) { + List<ResolveResult> result = new ArrayList<>(); - for (com.yahoo.search.federation.sourceref.Target target : searchChainResolver.defaultTargets()) { - try { - result.addData(target.responsibleSearchChain(sourceToProviderMap)); - } catch (UnresolvedSearchChainException e) { - result.addError(e); - } + for (var target : searchChainResolver.defaultTargets()) { + result.add(target.responsibleSearchChain(sourceToProviderMap)); } - return result.build(); + return List.copyOf(result); } diff --git a/vespajlib/src/main/java/com/yahoo/errorhandling/Results.java b/container-search/src/main/java/com/yahoo/search/federation/Results.java index 939d2276efc..7598a14f759 100644 --- a/vespajlib/src/main/java/com/yahoo/errorhandling/Results.java +++ b/container-search/src/main/java/com/yahoo/search/federation/Results.java @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.errorhandling; +package com.yahoo.search.federation; import java.util.ArrayList; import java.util.Collection; @@ -13,15 +13,11 @@ public class Results<DATA, ERROR> { private final List<DATA> data; private final List<ERROR> errors; - public Results(List<DATA> data, List<ERROR> errors) { + private Results(List<DATA> data, List<ERROR> errors) { this.data = List.copyOf(data); this.errors = List.copyOf(errors); } - public boolean hasErrors() { - return !errors.isEmpty(); - } - public List<DATA> data() { return data; } @@ -37,19 +33,10 @@ public class Results<DATA, ERROR> { public void addData(DATA d) { data.add(d); } - - public void addAllData(Collection<? extends DATA> d) { - data.addAll(d); - } - public void addError(ERROR e) { errors.add(e); } - public void addAllErrors(Collection<? extends ERROR> e) { - errors.addAll(e); - } - public Results<DATA, ERROR> build() { return new Results<>(data, errors); } diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/ResolveResult.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/ResolveResult.java new file mode 100644 index 00000000000..d9681140ae9 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/ResolveResult.java @@ -0,0 +1,14 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref; + +/** + * @author baldersheim + */ +public record ResolveResult(SearchChainInvocationSpec invocationSpec, String errorMsg) { + ResolveResult(SearchChainInvocationSpec invocationSpec) { + this(invocationSpec, null); + } + ResolveResult(String errorMsg) { + this(null, errorMsg); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java index 7dc65c819e4..9e45b6576a6 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java @@ -46,16 +46,6 @@ public class SearchChainResolver { public static class Builder { - public interface InvocationSpecFactory { - SearchChainInvocationSpec create(ComponentId searchChainId, FederationOptions federationOptions, List<String> schemas); - } - - private class DefaultInvocationSpecFactory implements InvocationSpecFactory { - public SearchChainInvocationSpec create(ComponentId searchChainId, FederationOptions federationOptions, List<String> schemas) { - return new SearchChainInvocationSpec(searchChainId, federationOptions, schemas); - } - } - private final SortedSet<Target> defaultTargets = new TreeSet<>(); private final ComponentRegistry<Target> targets = new ComponentRegistry<>() { @@ -137,19 +127,13 @@ public class SearchChainResolver { this.defaultTargets = Collections.unmodifiableSortedSet(defaultTargets); } - public SearchChainInvocationSpec resolve(ComponentSpecification sourceRef, Properties sourceToProviderMap) - throws UnresolvedSearchChainException { + public ResolveResult resolve(ComponentSpecification sourceRef, Properties sourceToProviderMap) { - Target target = resolveTarget(sourceRef); - return target.responsibleSearchChain(sourceToProviderMap); - } - - private Target resolveTarget(ComponentSpecification sourceRef) throws UnresolvedSearchChainException { Target target = targets.getComponent(sourceRef); if (target == null) { - throw UnresolvedSourceRefException.createForMissingSourceRef(sourceRef); + return new ResolveResult(SourceRefResolver.createForMissingSourceRef(sourceRef)); } - return target; + return target.responsibleSearchChain(sourceToProviderMap); } public SortedSet<Target> allTopLevelTargets() { diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java index 608566552cd..3de67908217 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java @@ -17,8 +17,8 @@ public class SingleTarget extends Target { } @Override - public SearchChainInvocationSpec responsibleSearchChain(Properties queryProperties) { - return searchChainInvocationSpec; + public ResolveResult responsibleSearchChain(Properties queryProperties) { + return new ResolveResult(searchChainInvocationSpec); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java index 2e7849dd85a..b5c40db01f8 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java @@ -4,10 +4,9 @@ package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentSpecification; import com.yahoo.processing.request.Properties; -import java.util.LinkedHashSet; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; /** * Maps a source reference to search chain invocation specs. @@ -24,21 +23,18 @@ public class SourceRefResolver { this.schema2Clusters = schema2Clusters; } - public Set<SearchChainInvocationSpec> resolve(ComponentSpecification sourceRef, - Properties sourceToProviderMap) throws UnresolvedSearchChainException { - try { - return Set.of(searchChainResolver.resolve(sourceRef, sourceToProviderMap)); - } catch (UnresolvedSourceRefException e) { + public List<ResolveResult> resolve(ComponentSpecification sourceRef, Properties sourceToProviderMap) { + ResolveResult searchChainResolveResult = searchChainResolver.resolve(sourceRef, sourceToProviderMap); + if (searchChainResolveResult.invocationSpec() == null) { return resolveClustersWithDocument(sourceRef, sourceToProviderMap); } + return List.of(searchChainResolveResult); } - private Set<SearchChainInvocationSpec> resolveClustersWithDocument(ComponentSpecification sourceRef, - Properties sourceToProviderMap) - throws UnresolvedSearchChainException { + private List<ResolveResult> resolveClustersWithDocument(ComponentSpecification sourceRef, Properties sourceToProviderMap) { if (hasOnlyName(sourceRef)) { - Set<SearchChainInvocationSpec> clusterSearchChains = new LinkedHashSet<>(); + List<ResolveResult> clusterSearchChains = new ArrayList<>(); List<String> clusters = schema2Clusters.getOrDefault(sourceRef.getName(), List.of()); for (String cluster : clusters) { @@ -48,21 +44,22 @@ public class SourceRefResolver { if ( ! clusterSearchChains.isEmpty()) return clusterSearchChains; } - throw UnresolvedSourceRefException.createForMissingSourceRef(sourceRef); + return List.of(new ResolveResult(createForMissingSourceRef(sourceRef))); } - private SearchChainInvocationSpec resolveClusterSearchChain(String cluster, - ComponentSpecification sourceRef, - Properties sourceToProviderMap) - throws UnresolvedSearchChainException { - try { - return searchChainResolver.resolve(new ComponentSpecification(cluster), sourceToProviderMap); - } - catch (UnresolvedSearchChainException e) { - throw new UnresolvedSearchChainException("Failed to resolve cluster search chain '" + cluster + - "' when using source ref '" + sourceRef + - "' as a document name."); + static String createForMissingSourceRef(ComponentSpecification source) { + return "Could not resolve source ref '" + source + "'."; + } + + private ResolveResult resolveClusterSearchChain(String cluster, + ComponentSpecification sourceRef, + Properties sourceToProviderMap) { + var resolveResult = searchChainResolver.resolve(new ComponentSpecification(cluster), sourceToProviderMap); + if (resolveResult.invocationSpec() == null) { + return new ResolveResult("Failed to resolve cluster search chain '" + cluster + + "' when using source ref '" + sourceRef + "' as a document name."); } + return resolveResult; } private boolean hasOnlyName(ComponentSpecification sourceSpec) { diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java index b6d99758c7b..a3c0328290d 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java @@ -16,7 +16,7 @@ import java.util.TreeSet; public class SourcesTarget extends Target { - private ComponentRegistry<ComponentAdaptor<SearchChainInvocationSpec>> providerSources = new ComponentRegistry<>() {}; + private final ComponentRegistry<ComponentAdaptor<SearchChainInvocationSpec>> providerSources = new ComponentRegistry<>() {}; private SearchChainInvocationSpec defaultProviderSource; @@ -25,10 +25,10 @@ public class SourcesTarget extends Target { } @Override - public SearchChainInvocationSpec responsibleSearchChain(Properties queryProperties) throws UnresolvedSearchChainException { + public ResolveResult responsibleSearchChain(Properties queryProperties) { ComponentSpecification providerSpecification = providerSpecificationForSource(queryProperties); if (providerSpecification == null) { - return defaultProviderSource; + return new ResolveResult(defaultProviderSource); } else { return lookupProviderSource(providerSpecification); } @@ -36,11 +36,7 @@ public class SourcesTarget extends Target { @Override public String searchRefDescription() { - StringBuilder builder = new StringBuilder(sourceId().stringValue()); - builder.append("[provider = "). - append(Joiner.on(", ").join(allProviderIdsStringValue())). - append("]"); - return builder.toString(); + return sourceId().stringValue() + "[provider = " + Joiner.on(", ").join(allProviderIdsStringValue()) + "]"; } private SortedSet<String> allProviderIdsStringValue() { @@ -51,14 +47,13 @@ public class SourcesTarget extends Target { return result; } - private SearchChainInvocationSpec lookupProviderSource(ComponentSpecification providerSpecification) - throws UnresolvedSearchChainException { + private ResolveResult lookupProviderSource(ComponentSpecification providerSpecification) { ComponentAdaptor<SearchChainInvocationSpec> providerSource = providerSources.getComponent(providerSpecification); if (providerSource == null) - throw UnresolvedProviderException.createForMissingProvider(sourceId(), providerSpecification); + return new ResolveResult("No provider '" + sourceId() + "' for source '" + providerSpecification + "'."); - return providerSource.model; + return new ResolveResult(providerSource.model); } public void freeze() { diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java index 38baf084d97..d35f7f7b181 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java @@ -23,9 +23,8 @@ public abstract class Target extends AbstractComponent { this(localId, false); } - public abstract SearchChainInvocationSpec responsibleSearchChain(Properties queryProperties) throws UnresolvedSearchChainException; + public abstract ResolveResult responsibleSearchChain(Properties queryProperties); public abstract String searchRefDescription(); abstract void freeze(); - } diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java deleted file mode 100644 index aa21ad3b369..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.sourceref; - -import com.yahoo.component.ComponentId; -import com.yahoo.component.ComponentSpecification; - -/** - * @author Tony Vaagenes - */ -@SuppressWarnings("serial") -class UnresolvedProviderException extends UnresolvedSearchChainException { - UnresolvedProviderException(String msg) { - super(msg); - } - - static UnresolvedSearchChainException createForMissingProvider(ComponentId source, - ComponentSpecification provider) { - return new UnresolvedProviderException("No provider '" + provider + "' for source '" + source + "'."); - } -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java deleted file mode 100644 index 0c8562e6032..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.sourceref; - -/** - * Thrown if a search chain can not be resolved from one or more ids. - * @author Tony Vaagenes - */ -public class UnresolvedSearchChainException extends Exception { - public UnresolvedSearchChainException(String msg) { - super(msg); - } -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java deleted file mode 100644 index fa2c1da13f0..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.sourceref; - -import com.yahoo.component.ComponentSpecification; - -/** - * @author Tony Vaagenes - */ -class UnresolvedSourceRefException extends UnresolvedSearchChainException { - UnresolvedSourceRefException(String msg) { - super(msg); - } - - - static UnresolvedSearchChainException createForMissingSourceRef(ComponentSpecification source) { - return new UnresolvedSourceRefException("Could not resolve source ref '" + source + "'."); - } -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java index d9046075f38..e5bbb48e807 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java @@ -13,8 +13,8 @@ import java.util.Iterator; import java.util.SortedSet; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.fail; /** * @author Tony Vaagenes @@ -59,37 +59,33 @@ public class SearchChainResolverTestCase { @Test void require_error_message_for_invalid_source() { - try { - resolve("no-such-source"); - fail("Expected exception."); - } catch (UnresolvedSearchChainException e) { - assertEquals("Could not resolve source ref 'no-such-source'.", e.getMessage()); - } + var result = resolve("no-such-source"); + assertEquals("Could not resolve source ref 'no-such-source'.", result.errorMsg()); } @Test - void lookup_search_chain() throws Exception { - SearchChainInvocationSpec res = resolve(searchChainId.getName()); + void lookup_search_chain() { + SearchChainInvocationSpec res = resolve(searchChainId.getName()).invocationSpec(); assertEquals(searchChainId, res.searchChainId); } //TODO: TVT: @Test() - public void lookup_provider() throws Exception { - SearchChainInvocationSpec res = resolve(providerId.getName()); + public void lookup_provider() { + SearchChainInvocationSpec res = resolve(providerId.getName()).invocationSpec(); assertEquals(providerId, res.provider); assertNull(res.source); assertEquals(providerId, res.searchChainId); } @Test - void lookup_source() throws Exception { - SearchChainInvocationSpec res = resolve(sourceId.getName()); + void lookup_source() { + SearchChainInvocationSpec res = resolve(sourceId.getName()).invocationSpec(); assertIsSourceInProvider(res); } @Test - void lookup_source_search_chain_directly() throws Exception { - SearchChainInvocationSpec res = resolve(sourceChainInProviderId.stringValue()); + void lookup_source_search_chain_directly() { + SearchChainInvocationSpec res = resolve(sourceChainInProviderId.stringValue()).invocationSpec(); assertIsSourceInProvider(res); } @@ -100,8 +96,8 @@ public class SearchChainResolverTestCase { } @Test - void lookup_source_for_provider2() throws Exception { - SearchChainInvocationSpec res = resolve(sourceId.getName(), provider2Id.getName()); + void lookup_source_for_provider2() { + SearchChainInvocationSpec res = resolve(sourceId.getName(), provider2Id.getName()).invocationSpec(); assertEquals(provider2Id, res.provider); assertEquals(sourceId, res.source); assertEquals(sourceChainInProvider2Id, res.searchChainId); @@ -126,22 +122,24 @@ public class SearchChainResolverTestCase { return new PropertyMap(); } - private SearchChainInvocationSpec resolve(String sourceSpecification) throws UnresolvedSearchChainException { + private ResolveResult resolve(String sourceSpecification) { return resolve(sourceSpecification, emptySourceToProviderMap()); } - private SearchChainInvocationSpec resolve(String sourceSpecification, String providerSpecification) - throws UnresolvedSearchChainException { + private ResolveResult resolve(String sourceSpecification, String providerSpecification) { Properties sourceToProviderMap = emptySourceToProviderMap(); sourceToProviderMap.set("source." + sourceSpecification + ".provider", providerSpecification); return resolve(sourceSpecification, sourceToProviderMap); } - private SearchChainInvocationSpec resolve(String sourceSpecification, Properties sourceToProviderMap) - throws UnresolvedSearchChainException { - SearchChainInvocationSpec res = searchChainResolver.resolve( + private ResolveResult resolve(String sourceSpecification, Properties sourceToProviderMap) { + ResolveResult res = searchChainResolver.resolve( ComponentSpecification.fromString(sourceSpecification), sourceToProviderMap); - assertEquals(federationOptions, res.federationOptions); + if (res.invocationSpec() != null) { + assertEquals(federationOptions, res.invocationSpec().federationOptions); + } else { + assertNotNull(res.errorMsg()); + } return res; } diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java index b32135afc94..95262937c01 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java @@ -3,21 +3,16 @@ package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.IndexModel; import com.yahoo.search.searchchain.model.federation.FederationOptions; import org.junit.jupiter.api.Test; -import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.TreeMap; import static com.yahoo.search.federation.sourceref.SearchChainResolverTestCase.emptySourceToProviderMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; /** * Test for SourceRefResolver. @@ -47,49 +42,38 @@ public class SourceRefResolverTestCase { @Test void lookup_search_chain() throws Exception { - Set<SearchChainInvocationSpec> searchChains = resolve(cluster1); + List<ResolveResult> searchChains = resolve(cluster1); assertEquals(1, searchChains.size()); assertTrue(searchChainIds(searchChains).contains(cluster1)); } @Test void lookup_search_chains_for_document1() throws Exception { - Set<SearchChainInvocationSpec> searchChains = resolve("document1"); + List<ResolveResult> searchChains = resolve("document1"); assertEquals(2, searchChains.size()); assertTrue(searchChainIds(searchChains).containsAll(List.of(cluster1, cluster2))); } @Test void error_when_document_gives_cluster_without_matching_search_chain() { - try { - resolve("document3"); - fail("Expected exception"); - } catch (UnresolvedSearchChainException e) { - assertEquals("Failed to resolve cluster search chain 'cluster3' " + - "when using source ref 'document3' as a document name.", - e.getMessage()); - } + List<ResolveResult> result = resolve("document3"); + + assertEquals("Failed to resolve cluster search chain 'cluster3' " + + "when using source ref 'document3' as a document name.", + result.get(0).errorMsg()); } @Test void error_when_no_document_or_search_chain() { - try { - resolve("document4"); - fail("Expected exception"); - } catch (UnresolvedSearchChainException e) { - assertEquals("Could not resolve source ref 'document4'.", e.getMessage()); - } + List<ResolveResult> results = resolve("document4"); + assertEquals("Could not resolve source ref 'document4'.", results.get(0).errorMsg()); } - private List<String> searchChainIds(Set<SearchChainInvocationSpec> searchChains) { - List<String> names = new ArrayList<>(); - for (SearchChainInvocationSpec searchChain : searchChains) { - names.add(searchChain.searchChainId.stringValue()); - } - return names; + private List<String> searchChainIds(Collection<ResolveResult> searchChains) { + return searchChains.stream().map(r -> r.invocationSpec().searchChainId.stringValue()).toList(); } - private Set<SearchChainInvocationSpec> resolve(String documentName) throws UnresolvedSearchChainException { + private List<ResolveResult> resolve(String documentName) { return sourceRefResolver.resolve(ComponentSpecification.fromString(documentName), emptySourceToProviderMap()); } diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index a3f8960e35f..15f33d67f84 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -68,7 +68,7 @@ <assertj.vespa.version>3.25.3</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <aws-sdk.vespa.version>1.12.703</aws-sdk.vespa.version> + <aws-sdk.vespa.version>1.12.705</aws-sdk.vespa.version> <athenz.vespa.version>1.11.56</athenz.vespa.version> <!-- Athenz END --> @@ -174,14 +174,14 @@ <maven-deploy-plugin.vespa.version>3.1.1</maven-deploy-plugin.vespa.version> <maven-enforcer-plugin.vespa.version>3.4.1</maven-enforcer-plugin.vespa.version> <maven-failsafe-plugin.vespa.version>3.2.5</maven-failsafe-plugin.vespa.version> - <maven-gpg-plugin.vespa.version>3.2.3</maven-gpg-plugin.vespa.version> + <maven-gpg-plugin.vespa.version>3.2.4</maven-gpg-plugin.vespa.version> <maven-install-plugin.vespa.version>3.1.1</maven-install-plugin.vespa.version> - <maven-jar-plugin.vespa.version>3.4.0</maven-jar-plugin.vespa.version> + <maven-jar-plugin.vespa.version>3.4.1</maven-jar-plugin.vespa.version> <maven-javadoc-plugin.vespa.version>3.6.3</maven-javadoc-plugin.vespa.version> <maven-plugin-api.vespa.version>${maven-core.vespa.version}</maven-plugin-api.vespa.version> <maven-plugin-tools.vespa.version>3.12.0</maven-plugin-tools.vespa.version> <maven-resources-plugin.vespa.version>3.3.1</maven-resources-plugin.vespa.version> - <maven-resolver.vespa.version>1.9.18</maven-resolver.vespa.version> + <maven-resolver.vespa.version>1.9.19</maven-resolver.vespa.version> <maven-shade-plugin.vespa.version>3.5.2</maven-shade-plugin.vespa.version> <maven-site-plugin.vespa.version>3.12.1</maven-site-plugin.vespa.version> <maven-source-plugin.vespa.version>3.3.1</maven-source-plugin.vespa.version> diff --git a/eval/src/vespa/eval/eval/typed_cells.h b/eval/src/vespa/eval/eval/typed_cells.h index 6cb8675cd5f..3dd8c30a3a9 100644 --- a/eval/src/vespa/eval/eval/typed_cells.h +++ b/eval/src/vespa/eval/eval/typed_cells.h @@ -36,6 +36,7 @@ struct TypedCells { TypedCells(const TypedCells &other) noexcept = default; TypedCells & operator= (TypedCells &&other) noexcept = default; TypedCells & operator= (const TypedCells &other) noexcept = default; + bool valid() const noexcept { return size != 0; } }; } // namespace diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 558b8dea8d9..400934fce97 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -423,7 +423,7 @@ public class Flags { "Takes effect immediately"); public static UnboundBooleanFlag NEW_PATH_FOR_DISK_ENCRYPTION_KEY_METADATA = defineFeatureFlag( - "new-path-for-disk-encryption-key-metadata", false, + "new-path-for-disk-encryption-key-metadata", true, List.of("hmusum"), "2024-04-08", "2024-06-01", "Whether to read and write disk encryption key to new path", "Will be read only on boot."); diff --git a/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java b/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java index fd1b8b700c8..aa7c071b93a 100644 --- a/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java +++ b/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java @@ -9,7 +9,6 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.component.annotation.Inject; import de.kherud.llama.LlamaModel; import de.kherud.llama.ModelParameters; -import de.kherud.llama.args.LogFormat; import java.util.ArrayList; import java.util.List; @@ -43,7 +42,7 @@ public class LocalLLM extends AbstractComponent implements LanguageModel { maxTokens = config.maxTokens(); // Only used if GPU is not used - var defaultThreadCount = Runtime.getRuntime().availableProcessors() - 2; + var defaultThreadCount = Math.max(Runtime.getRuntime().availableProcessors() - 2, 1); var modelFile = config.model().toFile().getAbsolutePath(); var modelParams = new ModelParameters() diff --git a/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java b/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java index a3b260f3fb5..95bcfb985bd 100644 --- a/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java +++ b/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java @@ -6,7 +6,6 @@ import ai.vespa.llm.completion.Completion; import ai.vespa.llm.completion.Prompt; import ai.vespa.llm.completion.StringPrompt; import com.yahoo.config.ModelReference; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -33,10 +32,10 @@ public class LocalLLMTest { private static Prompt prompt = StringPrompt.from("A random prompt"); @Test - @Disabled public void testGeneration() { var config = new LlmLocalClientConfig.Builder() .parallelRequests(1) + .threads(1) .model(ModelReference.valueOf(model)); var llm = new LocalLLM(config.build()); @@ -50,12 +49,12 @@ public class LocalLLMTest { } @Test - @Disabled public void testAsyncGeneration() { var sb = new StringBuilder(); var tokenCount = new AtomicInteger(0); var config = new LlmLocalClientConfig.Builder() .parallelRequests(1) + .threads(1) .model(ModelReference.valueOf(model)); var llm = new LocalLLM(config.build()); @@ -78,7 +77,6 @@ public class LocalLLMTest { } @Test - @Disabled public void testParallelGeneration() { var prompts = testPrompts(); var promptsToUse = prompts.size(); @@ -90,6 +88,7 @@ public class LocalLLMTest { var config = new LlmLocalClientConfig.Builder() .parallelRequests(parallelRequests) + .threads(1) .model(ModelReference.valueOf(model)); var llm = new LocalLLM(config.build()); @@ -117,7 +116,6 @@ public class LocalLLMTest { } @Test - @Disabled public void testRejection() { var prompts = testPrompts(); var promptsToUse = prompts.size(); @@ -130,6 +128,7 @@ public class LocalLLMTest { var config = new LlmLocalClientConfig.Builder() .parallelRequests(parallelRequests) + .threads(1) .maxQueueSize(additionalQueue) .model(ModelReference.valueOf(model)); var llm = new LocalLLM(config.build()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java index 940109bab8a..8b2dc44669f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.collections.Pair; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; @@ -33,6 +34,11 @@ public class MemoryMetricsDb implements MetricsDb { /** Lock all access for now since we modify lists inside a map */ private final Object lock = new Object(); + @Inject + public MemoryMetricsDb() { + this(Clock.systemUTC()); + } + public MemoryMetricsDb(Clock clock) { this.clock = clock; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index a1306d7831a..ec48e1d5f34 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -51,6 +51,8 @@ public class CapacityPolicies { private ClusterResources applyOn(ClusterResources resources, Capacity capacity, ApplicationId application, boolean exclusive) { int nodes = decideSize(resources.nodes(), capacity.isRequired(), application.instance().isTester()); int groups = Math.min(resources.groups(), nodes); // cannot have more groups than nodes + while (groups > 1 && nodes % groups != 0) + groups--; // Must be divisible by the number of groups var nodeResources = decideNodeResources(resources.nodeResources(), capacity.isRequired(), exclusive); return new ClusterResources(nodes, groups, nodeResources); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 8c52f389daf..b149a9af2c2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -179,6 +179,13 @@ class NodeAllocation { if (violatesExclusivity(candidate) != NodeCandidate.ExclusivityViolation.NONE) return Retirement.violatesExclusivity; if (requiredHostFlavor.isPresent() && ! candidate.parent.map(node -> node.flavor().name()).equals(requiredHostFlavor)) return Retirement.violatesHostFlavor; if (candidate.violatesSpares) return Retirement.violatesSpares; + + var group = candidate.allocation().get().membership().cluster().group(); + if (cluster.isStateful() && group.isPresent() && requested.count().isPresent()) { + long nodesInGroup = nodes.values().stream().filter(n -> groupOf(n).equals(group) && ! isRetired(n)).count(); + if (nodesInGroup >= requested.groupSize()) + return Retirement.groupSurplus; + } return Retirement.none; } @@ -290,6 +297,10 @@ class NodeAllocation { return candidate.allocation().flatMap(a -> a.membership().cluster().group()); } + private boolean isRetired(NodeCandidate candidate) { + return candidate.allocation().map(a -> a.membership().retired()).orElse(false); + } + private Node resize(Node node) { NodeResources hostResources = allNodes.parentOf(node).get().flavor().resources(); return node.with(new Flavor(requested.resources().get() @@ -463,6 +474,7 @@ class NodeAllocation { violatesHostFlavor("node violates host flavor"), violatesHostFlavorGeneration("node violates host flavor generation"), violatesSpares("node is assigned to a host we want to use as a spare"), + groupSurplus("group has enough nodes"), none(""); private final String description; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index d4c4e86f0a3..d8565b81e41 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -55,6 +55,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat final boolean exclusiveSwitch; /** True if this node belongs to a group which will not be needed after this deployment */ + // TODO: Always false final boolean isSurplus; /** This node does not exist in the node repository yet */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index f653416d973..81a490939d6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -27,6 +27,7 @@ public class ContainerConfig { <accesslog type='disabled'/> <component id='com.yahoo.test.ManualClock'/> <component id='com.yahoo.vespa.curator.mock.MockCurator'/> + <component id='com.yahoo.vespa.hosted.provision.autoscale.MemoryMetricsDb'/> <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockDeployer'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockInfraDeployer'/> @@ -34,7 +35,6 @@ public class ContainerConfig { <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockDuperModel'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/> - <component id='com.yahoo.vespa.hosted.provision.autoscale.QuestMetricsDb'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockMetricsFetcher'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider'/> diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java index ff5ffd82bf1..a91902c8eba 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java @@ -399,7 +399,7 @@ public class DynamicAllocationTest { } @Test - public void node_resources_are_relaxed_in_dev() { + public void node_resources_are_reduced_in_dev() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); @@ -419,6 +419,22 @@ public class DynamicAllocationTest { } @Test + public void node_resources_are_reduced_in_staging() { + var resources = new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.staging, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); + tester.makeReadyNodes(3, new Flavor(resources), NodeType.host, 10, true); + tester.activateTenantHosts(); + + ApplicationId application = ProvisioningTester.applicationId(); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("1").build(); + + List<HostSpec> hosts = tester.prepare(application, cluster, 36, 2, resources); + tester.activate(application, hosts); + assertEquals(3, hosts.size()); + assertEquals(1, hosts.stream().map(host -> host.membership().get().cluster().group().get()).distinct().count()); + } + + @Test public void switching_from_legacy_flavor_syntax_to_resources_does_not_cause_reallocation() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(5, 20, 1400, 3)), NodeType.host, 10, true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index abcef421b4c..78a34326949 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -553,6 +553,34 @@ public class DynamicProvisioningTest { 2, 1, resources); } + @Test + public void split_into_two_groups() { + List<Flavor> flavors = List.of(new Flavor("2x", new NodeResources(2, 20, 200, 0.1, fast, local))); + + ProvisioningTester tester = new ProvisioningTester.Builder().dynamicProvisioning(true, false) + .flavors(flavors) + .hostProvisioner(new MockHostProvisioner(flavors)) + .nameResolver(nameResolver) + .build(); + + tester.activateTenantHosts(); + + ApplicationId app1 = applicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(content, new ClusterSpec.Id("cluster1")).vespaVersion("8").build(); + + System.out.println("Initial deployment ----------------------"); + tester.activate(app1, cluster1, Capacity.from(resources(6, 1, 2, 20, 200, fast, StorageType.any))); + tester.assertNodes("Initial deployment: 1 group", + 6, 1, 2, 20, 200, fast, remote, app1, cluster1); + + System.out.println("Split into 2 groups ---------------------"); + tester.activate(app1, cluster1, Capacity.from(resources(6, 2, 2, 20, 200, fast, StorageType.any))); + tester.assertNodes("Change to 2 groups: Gets 6 active non-retired nodes", + 6, 2, 2, 20, 200, fast, remote, app1, cluster1); + List<Node> retired = tester.nodeRepository().nodes().list().owner(app1).cluster(cluster1.id()).state(Node.State.active).retired().asList(); + assertEquals("... and in addition 3 retired nodes", 3, retired.size()); + } + private ProvisioningTester tester(boolean sharing) { var hostProvisioner = new MockHostProvisioner(new NodeFlavors(ProvisioningTester.createConfig()).getFlavors(), nameResolver, 0); return new ProvisioningTester.Builder().dynamicProvisioning(true, sharing).hostProvisioner(hostProvisioner).nameResolver(nameResolver).build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 5f2790e886a..7b690b880c2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -288,13 +288,13 @@ public class ProvisioningTest { assertEquals("Superfluous container nodes are also dirtyed", 4-2 + 5-2 + 1 + 4-2, tester.nodeRepository().nodes().list(Node.State.dirty).size()); assertEquals("Superfluous content nodes are retired", - 5-3 + 6-3 - 1, tester.getNodes(application1, Node.State.active).retired().size()); + 5-3 + 6-3 -1, tester.getNodes(application1, Node.State.active).retired().size()); // increase content slightly SystemState state6 = prepare(application1, 2, 2, 4, 3, defaultResources, tester); tester.activate(application1, state6.allHosts); assertEquals("One content node is unretired", - 5-4 + 6-3 - 1, tester.getNodes(application1, Node.State.active).retired().size()); + 5-4 + 6-3 -1, tester.getNodes(application1, Node.State.active).retired().size()); // Then reserve more SystemState state7 = prepare(application1, 8, 2, 2, 2, defaultResources, tester); @@ -505,7 +505,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(31, hostFlavor.resources()).activateTenantHosts(); + tester.makeReadyHosts(32, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.applicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp b/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp index f7a358efb26..f4a1ade8a66 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp +++ b/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp @@ -750,19 +750,36 @@ void run_intermediate_blueprint_benchmark(const BlueprintFactorySetup& a, const BlueprintFactorySetup& b, size_t num_docs) { print_intermediate_blueprint_result_header(2); + double max_speedup = 0.0; + double min_speedup = std::numeric_limits<double>::max(); for (double b_hit_ratio: b.op_hit_ratios) { auto b_factory = b.make_factory_shared(num_docs, b_hit_ratio); for (double a_hit_ratio : a.op_hit_ratios) { IntermediateBlueprintFactoryType factory; factory.add_child(a.make_factory(num_docs, a_hit_ratio)); factory.add_child(b_factory); - for (auto algo: {PlanningAlgo::Order, PlanningAlgo::Estimate, PlanningAlgo::Cost, PlanningAlgo::CostForceStrict}) { + double time_ms_esti = 0.0; + for (auto algo: {PlanningAlgo::Order, PlanningAlgo::Estimate, PlanningAlgo::Cost, + PlanningAlgo::CostForceStrict}) { auto res = benchmark_search(factory, num_docs + 1, true, false, false, 1.0, algo); print_intermediate_blueprint_result(res, {a_hit_ratio, b_hit_ratio}, algo, num_docs); + if (algo == PlanningAlgo::Estimate) { + time_ms_esti = res.time_ms; + } + if (algo == PlanningAlgo::CostForceStrict) { + double speedup = time_ms_esti / res.time_ms; + if (speedup > max_speedup) { + max_speedup = speedup; + } + if (speedup < min_speedup) { + min_speedup = speedup; + } + std::cout << "speedup (esti/forc)=" << std::setprecision(4) << speedup << std::endl; + } } - std::cout << std::endl; } } + std::cout << "max_speedup=" << max_speedup << ", min_speedup=" << min_speedup << std::endl << std::endl; } void @@ -787,6 +804,12 @@ gen_ratios(double middle, double range_multiplier, size_t num_samples) for (size_t i = 0; i < num_samples; ++i) { res.push_back(ratio); ratio *= factor; + if (ratio > 1.0) { + if (res.size() < num_samples) { + res.push_back(1.0); + } + break; + } } return res; } @@ -831,7 +854,6 @@ TEST(IteratorBenchmark, analyze_term_search_in_disk_index) { BenchmarkSetup setup(num_docs, {str_index}, {QueryOperator::Term}, {true, false}, base_hit_ratios); setup.filter_hit_ratios = filter_hit_ratios; - setup.filter_crossover_factor = 1.0; run_benchmarks(setup, global_summary); } @@ -863,31 +885,24 @@ TEST(IteratorBenchmark, analyze_term_search_in_fast_search_attributes) run_benchmarks(setup, global_summary); } -TEST(IteratorBenchmark, analyze_in_operator_non_strict) +TEST(IteratorBenchmark, analyze_IN_non_strict) { - const std::vector<double> hit_ratios = {0.001, 0.01, 0.1, 0.2, 0.4, 0.6, 0.8}; - BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::In}, {false}, hit_ratios, {5, 9, 10, 100, 1000, 10000}); - setup.disjunct_children = true; - run_benchmarks(setup); + for (auto in_hit_ratio : {0.01, 0.1, 0.5}) { + BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::In}, {false}, {in_hit_ratio}, {2, 5, 9, 10, 100, 1000, 10000}); + setup.filter_hit_ratios = gen_ratios(in_hit_ratio, 10.0, 13); + setup.disjunct_children = true; + run_benchmarks(setup); + } } -TEST(IteratorBenchmark, analyze_in_operator_strict) +TEST(IteratorBenchmark, analyze_IN_strict) { const std::vector<double> hit_ratios = {0.001, 0.01, 0.1, 0.2, 0.4, 0.6, 0.8}; - BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::In}, {true}, hit_ratios, {5, 9, 10, 100, 1000, 10000}); + BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::In}, {true}, hit_ratios, {2, 5, 9, 10, 100, 1000, 10000}); setup.disjunct_children = true; run_benchmarks(setup); } -TEST(IteratorBenchmark, analyze_complex_leaf_operators) -{ - std::vector<FieldConfig> field_cfgs = {int32_array_fs}; - std::vector<QueryOperator> query_ops = {QueryOperator::In, QueryOperator::DotProduct}; - const std::vector<double> hit_ratios = {0.001, 0.01, 0.1, 0.2, 0.4, 0.6, 0.8}; - BenchmarkSetup setup(num_docs, field_cfgs, query_ops, {true, false}, hit_ratios, {1, 2, 10, 100}); - run_benchmarks(setup); -} - TEST(IteratorBenchmark, analyze_weak_and_operators) { std::vector<FieldConfig> field_cfgs = {int32_wset_fs}; @@ -897,24 +912,6 @@ TEST(IteratorBenchmark, analyze_weak_and_operators) run_benchmarks(setup); } -TEST(IteratorBenchmark, term_benchmark) -{ - BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::Term}, {true, false}, base_hit_ratios); - run_benchmarks(setup); -} - -TEST(IteratorBenchmark, and_benchmark) -{ - BenchmarkSetup setup(num_docs, {int32_array_fs}, {QueryOperator::And}, {true, false}, base_hit_ratios, {1, 2, 4, 8}); - run_benchmarks(setup); -} - -TEST(IteratorBenchmark, or_benchmark) -{ - BenchmarkSetup setup(num_docs, {int32_array_fs}, {QueryOperator::Or}, {true, false}, base_hit_ratios, {1, 10, 100, 1000}); - run_benchmarks(setup); -} - TEST(IteratorBenchmark, or_vs_filter_crossover) { auto fixed_or = make_blueprint_factory(int32_array_fs, QueryOperator::Or, num_docs, 0, 0.1, 100, false); @@ -933,16 +930,38 @@ TEST(IteratorBenchmark, or_vs_filter_crossover_with_allow_force_strict) analyze_crossover(*fixed_or, variable_term, num_docs + 1, true, 0.0001); } -TEST(IteratorBenchmark, analyze_and_with_filter_vs_in) +TEST(IteratorBenchmark, analyze_AND_filter_vs_IN) +{ + for (auto in_filter_ratio : {0.01, 0.1, 0.5}) { + for (uint32_t children: {2, 10, 100, 1000}) { + run_and_benchmark({int32_fs, QueryOperator::Term, gen_ratios(in_filter_ratio, 10.0, 13)}, + {int32_fs, QueryOperator::In, {in_filter_ratio}, children, false}, + num_docs); + } + } +} + +TEST(IteratorBenchmark, analyze_AND_filter_vs_OR) +{ + for (auto or_filter_ratio : {0.01, 0.1, 0.5}) { + for (uint32_t children: {2, 10, 100, 1000}) { + run_and_benchmark({int32_fs, QueryOperator::Term, gen_ratios(or_filter_ratio, 10, 13)}, + {int32_fs, QueryOperator::Or, {or_filter_ratio}, children, false}, + num_docs); + } + } +} + +TEST(IteratorBenchmark, analyze_AND_filter_vs_IN_array) { - for (uint32_t children: {10, 100, 1000}) { - run_and_benchmark({int32_fs, QueryOperator::Term, gen_ratios(0.1, 8.0, 15)}, - {int32_fs, QueryOperator::In, {0.1}, children, false}, + for (uint32_t children: {2, 10, 100, 1000}) { + run_and_benchmark({int32_fs, QueryOperator::Term, gen_ratios(0.1, 10.0, 13)}, + {int32_array_fs, QueryOperator::In, {0.1}, children, false}, num_docs); } } -TEST(IteratorBenchmark, analyze_and_with_bitvector_vs_in) +TEST(IteratorBenchmark, analyze_AND_bitvector_vs_IN) { for (uint32_t children: {10, 100, 1000, 10000}) { run_and_benchmark({int32_fs, QueryOperator::In, {0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.40, 0.45, 0.50, 0.55, 0.60}, children, true}, @@ -951,21 +970,35 @@ TEST(IteratorBenchmark, analyze_and_with_bitvector_vs_in) } } -TEST(IteratorBenchmark, analyze_and_with_filter_vs_in_array) +TEST(IteratorBenchmark, analyze_OR_non_strict_fs) { - for (uint32_t children: {10, 100, 1000}) { - run_and_benchmark({int32_fs, QueryOperator::Term, gen_ratios(0.1, 8.0, 15)}, - {int32_array_fs, QueryOperator::In, {0.1}, children, false}, - num_docs); + for (auto or_hit_ratio : {0.01, 0.1, 0.5}) { + BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::Or}, {false}, {or_hit_ratio}, + {2, 4, 6, 8, 10, 100, 1000}); + setup.filter_hit_ratios = gen_ratios(or_hit_ratio, 10.0, 13); + run_benchmarks(setup); } } -TEST(IteratorBenchmark, analyze_and_with_filter_vs_or) +TEST(IteratorBenchmark, analyze_OR_non_strict_non_fs) { - for (uint32_t children: {10, 100, 1000}) { - run_and_benchmark({int32_fs, QueryOperator::Term, gen_ratios(0.1, 8.0, 15)}, - {int32_fs, QueryOperator::Or, {0.1}, children, false}, - num_docs); + BenchmarkSetup setup(num_docs, {int32}, {QueryOperator::Or}, {false}, {0.1}, {2, 4, 6, 8, 10}); + setup.filter_hit_ratios = gen_ratios(0.1, 10.0, 13); + run_benchmarks(setup); +} + +TEST(IteratorBenchmark, analyze_OR_strict) +{ + BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::Or}, {true}, {0.01, 0.1, 0.5}, {2, 4, 6, 8, 10, 100, 1000}); + run_benchmarks(setup); +} + +TEST(IteratorBenchmark, analyze_btree_iterator_non_strict) +{ + for (auto term_ratio : {0.01, 0.1, 0.5, 1.0}) { + BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::Term}, {false}, {term_ratio}, {1}); + setup.filter_hit_ratios = gen_ratios(term_ratio, 10.0, 15); + run_benchmarks(setup); } } diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index 391e2d91d08..eeae12e1695 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -20,6 +20,16 @@ using search::attribute::DistanceMetric; template <typename T> TypedCells t(const std::vector<T> &v) { return TypedCells(v); } +template<typename T> +struct EmptyCells { + explicit EmptyCells(size_t elems) : _zero(elems, 0), cells(_zero) { cells.size = 0; } + std::vector<T> _zero; + TypedCells cells; +}; + +template <typename T> +EmptyCells<T> e(size_t elems) { return EmptyCells<T>(elems); } + void verify_geo_miles(const std::vector<double> &p1, const std::vector<double> &p2, double exp_miles) @@ -49,6 +59,15 @@ void verify_geo_miles(const std::vector<double> &p1, } } +template<typename T> +void verifyInvalidQueryVector(DistanceFunctionFactory & dff, double expected_distance_to_origo) { + std::vector<T> origo = {0,0,0}; + EXPECT_FLOAT_EQ(expected_distance_to_origo, dff.for_query_vector(t(origo))->calc(e<double>(origo.size()).cells)); + EXPECT_FLOAT_EQ(expected_distance_to_origo, dff.for_query_vector(t(origo))->calc(e<float>(origo.size()).cells)); + EXPECT_FLOAT_EQ(expected_distance_to_origo, dff.for_query_vector(t(origo))->calc(e<Int8Float>(origo.size()).cells)); + EXPECT_FLOAT_EQ(expected_distance_to_origo, dff.for_query_vector(t(origo))->calc(e<vespalib::BFloat16>(origo.size()).cells)); +} + double computeEuclideanChecked(TypedCells a, TypedCells b) { static EuclideanDistanceFunctionFactory<Int8Float> i8f_dff; static EuclideanDistanceFunctionFactory<float> flt_dff; @@ -92,6 +111,7 @@ TEST(DistanceFunctionsTest, euclidean_gives_expected_score) EXPECT_EQ(d12, 2.0); EuclideanDistanceFunctionFactory<double> dff; + verifyInvalidQueryVector<double>(dff, 0.0); auto euclid = dff.for_query_vector(t(p0)); EXPECT_DOUBLE_EQ(euclid->to_rawscore(d12), 1.0/(1.0 + sqrt(2.0))); double threshold = euclid->convert_threshold(8.0); @@ -128,10 +148,7 @@ TEST(DistanceFunctionsTest, euclidean_gives_expected_score) EXPECT_EQ(computeEuclideanChecked(t(p6), t(p6)), 0.0); // smoke test for bfloat16: - std::vector<vespalib::BFloat16> bf16v; - bf16v.emplace_back(1.0); - bf16v.emplace_back(1.0); - bf16v.emplace_back(1.0); + std::vector<vespalib::BFloat16> bf16v{1.0, 1.0, 1.0}; EXPECT_EQ(computeEuclideanChecked(t(bf16v), t(p0)), 3.0); EXPECT_EQ(computeEuclideanChecked(t(bf16v), t(p1)), 2.0); EXPECT_EQ(computeEuclideanChecked(t(bf16v), t(p2)), 2.0); @@ -188,6 +205,7 @@ TEST(DistanceFunctionsTest, angular_gives_expected_score) AngularDistanceFunctionFactory<double> dff; auto angular = dff.for_query_vector(t(p0)); + verifyInvalidQueryVector<double>(dff, 1.0); constexpr double pi = 3.14159265358979323846; double a12 = computeAngularChecked(t(p1), t(p2)); double a13 = computeAngularChecked(t(p1), t(p3)); @@ -315,6 +333,7 @@ TEST(DistanceFunctionsTest, prenormalized_angular_gives_expected_score) std::vector<double> p8{3.0, 0.0, 0.0}; PrenormalizedAngularDistanceFunctionFactory<double> dff; + verifyInvalidQueryVector<double>(dff, 1.0); auto pnad = dff.for_query_vector(t(p0)); double i12 = computePrenormalizedAngularChecked(t(p1), t(p2)); @@ -360,7 +379,8 @@ TEST(DistanceFunctionsTest, prenormalized_angular_gives_expected_score) TEST(DistanceFunctionsTest, hamming_gives_expected_score) { - static HammingDistanceFunctionFactory<double> dff; + HammingDistanceFunctionFactory<double> dff; + verifyInvalidQueryVector<double>(dff, 0.0); std::vector<std::vector<double>> points{{0.0, 0.0, 0.0}, {1.0, 0.0, 0.0}, @@ -376,6 +396,7 @@ TEST(DistanceFunctionsTest, hamming_gives_expected_score) EXPECT_EQ(h0, 0.0); EXPECT_EQ(dist_fun->to_rawscore(h0), 1.0); } + double d12 = dff.for_query_vector(t(points[1]))->calc(t(points[2])); EXPECT_EQ(d12, 3.0); EXPECT_DOUBLE_EQ(hamming->to_rawscore(d12), 1.0/(1.0 + 3.0)); @@ -579,6 +600,9 @@ TEST(DistanceFunctionsTest, transformed_mips_basic_scores) std::vector<double> p4{0.5, 0.5, sq_root_half}; std::vector<double> p5{0.0,-1.0, 0.0}; + MipsDistanceFunctionFactory<double> dff; + verifyInvalidQueryVector<double>(dff, 0.0); + double i12 = computeTransformedMipsChecked(t(p1), t(p2)); double i13 = computeTransformedMipsChecked(t(p1), t(p3)); double i23 = computeTransformedMipsChecked(t(p2), t(p3)); diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index b697effeab4..fcce2f5eb17 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -62,8 +62,17 @@ public: _vectors[docid] = vec; return *this; } + void clear(uint32_t docid) { + if (docid < _vectors.size()) { + _vectors[docid].clear(); + } + } vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const noexcept override { - return get_vectors(docid).cells(subspace); + auto bundle = get_vectors(docid); + if (subspace < bundle.subspaces()) { + return bundle.cells(subspace); + } + return { nullptr, _subspace_type.cell_type(), 0 }; } VectorBundle get_vectors(uint32_t docid) const noexcept override { ArrayRef ref(_vectors[docid]); @@ -277,6 +286,12 @@ public: return index->get_active_nodes(); } + /* + * Simulate race where writer has cleared a tensor while read thread still + * use old graph. + */ + void writer_clears_tensor(uint32_t docid) { vectors.clear(docid); } + static constexpr bool is_single = std::is_same_v<IndexType, HnswIndex<HnswIndexType::SINGLE>>; }; @@ -827,6 +842,14 @@ TYPED_TEST(HnswIndexTest, hnsw_graph_can_be_saved_and_loaded) this->check_savetest_index("after load"); } +TYPED_TEST(HnswIndexTest, search_during_remove) +{ + this->init(false); + this->make_savetest_index(); + this->writer_clears_tensor(4); + this->expect_top_3_by_docid("{0, 0}", {0, 0}, {7}); +} + using HnswMultiIndexTest = HnswIndexTest<HnswIndex<HnswIndexType::MULTI>>; namespace { @@ -1022,4 +1045,15 @@ TYPED_TEST(TwoPhaseTest, two_phase_add) this->expect_levels(nodeids[0], {{2}, {4}}); } +TYPED_TEST(TwoPhaseTest, prepare_insert_during_remove) +{ + this->init(false); + this->make_savetest_index(); + this->writer_clears_tensor(4); + auto prepared = this->prepare_add(2, 1); + this->remove_document(4); + this->complete_add(2, std::move(prepared)); + EXPECT_EQ(2, this->get_active_nodes()); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index aebfda8fffd..569edc0a4ab 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -166,7 +166,7 @@ public: return {0.5, lookup_cost(indirections), lookup_strict_cost(indirections)}; } else { double rel_est = abs_to_rel_est(_hit_estimate.est_hits(), docid_limit); - return {rel_est, btree_cost(), btree_strict_cost(rel_est)}; + return {rel_est, btree_cost(rel_est), btree_strict_cost(rel_est)}; } } @@ -519,8 +519,9 @@ public: double estimate(const IDirectPostingStore::LookupResult &term) const noexcept { return abs_to_rel_est(term.posting_size, docid_limit); } - double cost(const IDirectPostingStore::LookupResult &) const noexcept { - return btree_cost(); + double cost(const IDirectPostingStore::LookupResult &term) const noexcept { + double rel_est = abs_to_rel_est(term.posting_size, docid_limit); + return btree_cost(rel_est); } double strict_cost(const IDirectPostingStore::LookupResult &term) const noexcept { double rel_est = abs_to_rel_est(term.posting_size, docid_limit); diff --git a/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.hpp b/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.hpp index 160bb199fb8..4ede8957f4e 100644 --- a/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.hpp +++ b/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.hpp @@ -192,8 +192,9 @@ DirectMultiTermBlueprint<PostingStoreType, SearchType>::calculate_flow_stats(uin double estimate(const IDirectPostingStore::LookupResult &term) const noexcept { return abs_to_rel_est(term.posting_size, docid_limit); } - double cost(const IDirectPostingStore::LookupResult &) const noexcept { - return search::queryeval::flow::btree_cost(); + double cost(const IDirectPostingStore::LookupResult &term) const noexcept { + double rel_est = abs_to_rel_est(term.posting_size, docid_limit); + return search::queryeval::flow::btree_cost(rel_est); } double strict_cost(const IDirectPostingStore::LookupResult &term) const noexcept { double rel_est = abs_to_rel_est(term.posting_size, docid_limit); diff --git a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp index c0d0b1baa8c..c30b02a0c37 100644 --- a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp @@ -72,7 +72,7 @@ queryeval::FlowStats DiskTermBlueprint::calculate_flow_stats(uint32_t docid_limit) const { double rel_est = abs_to_rel_est(_lookupRes->counts._numDocs, docid_limit); - return {rel_est, disk_index_cost(), disk_index_strict_cost(rel_est)}; + return {rel_est, disk_index_cost(rel_est), disk_index_strict_cost(rel_est)}; } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index eba135d8519..2bc94073c92 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -261,7 +261,7 @@ public: queryeval::FlowStats calculate_flow_stats(uint32_t docid_limit) const override { double rel_est = abs_to_rel_est(_posting_itr.size(), docid_limit); - return {rel_est, btree_cost(), btree_strict_cost(rel_est)}; + return {rel_est, btree_cost(rel_est), btree_strict_cost(rel_est)}; } SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray& tfmda) const override { diff --git a/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h b/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h index 356ecd4c992..e8a58bba9dc 100644 --- a/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h +++ b/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h @@ -7,60 +7,87 @@ namespace search::queryeval::flow { /** - * This function is used when calculating the strict cost of - * intermediate and complex leaf blueprints that use a heap for their strict iterator implementation. - * - * Iterator benchmarking has shown the need to increase the strict cost - * of complex blueprints, to avoid that they are forced strict too early. - * The 5.0 multiplier reflects this. - * - * Program used: searchlib/src/tests/queryeval/iterator_benchmark - * Tests used: analyze_and_with_filter_vs_* - */ -inline double heap_cost(double my_est, size_t num_children) { - return 5.0 * my_est * std::log2(std::max(size_t(1), num_children)); -} - -/** - * The following constants and formulas were derived after analyzing term search over attributes - * (with and without fast-search) and disk index by using the iterator benchmark program: + * The following constants and formulas were derived after benchmarking and analyzing + * using the following benchmark program: * searchlib/src/tests/queryeval/iterator_benchmark * - * The following tests were executed on a machine with an Intel Xeon 2.5 GHz CPU with 48 cores and 256 Gb of memory: - * ./searchlib_iterator_benchmark_test_app --gtest_filter='*analyze_term_search*' - * - * TODO: Add details on OR benchmarking. + * The tests were executed on: + * - Machine with an Intel Xeon 2.5 GHz CPU with 48 cores and 256 Gb of memory. + * - Apple M1 MacBook Pro (2021) 32 GB. * * The benchmark summary shows the 'average ms per cost' of the different benchmark cases. - * The following constants and formulas were derived to balance 'average ms per cost' to be similar + * The constants and formulas are adjusted to balance 'average ms per cost' to be similar * across the different benchmark cases. + * + * The AND benchmark cases outputs the ratio (esti/forc) of the time_ms used by two query planning algorithms: + * 'estimate' (legacy) and 'cost with allowed force strict' (new). + * 'max_speedup' indicates the gain of using the new cost model, while 'min_speedup' indicates the loss. + * The constants and formulas are also adjusted to maximize speedup, while reducing loss. + * Tests used: + * - IteratorBenchmark::analyze_AND_filter_vs_IN + * - IteratorBenchmark::analyze_AND_filter_vs_OR + * - IteratorBenchmark::analyze_AND_filter_vs_IN_array + * - IteratorBenchmark::analyze_AND_bitvector_vs_IN */ +/** + * This function is used when calculating the strict cost of + * intermediate and complex leaf blueprints that use a heap for their strict iterator implementation. + * Tests used: + * - IteratorBenchmark::analyze_IN_strict + * - IteratorBenchmark::analyze_OR_strict + */ +inline double heap_cost(double my_est, size_t num_children) { + return my_est * std::log2(std::max(size_t(1), num_children)); +} + // Non-strict cost of lookup based matching in an attribute (not fast-search). +// Test used: IteratorBenchmark::analyze_term_search_in_attributes_non_strict inline double lookup_cost(size_t num_indirections) { return 1.0 + (num_indirections * 1.0); } // Non-strict cost of reverse lookup into a hash table (containing terms from a multi-term operator). +// Test used: IteratorBenchmark::analyze_IN_non_strict inline double reverse_hash_lookup() { - return 5.0; + return 1.0; } // Strict cost of lookup based matching in an attribute (not fast-search). +// IteratorBenchmark::analyze_term_search_in_attributes_strict inline double lookup_strict_cost(size_t num_indirections) { return lookup_cost(num_indirections); } -// Non-strict cost of matching in a btree posting list (e.g. fast-search attribute or memory index field). -inline double btree_cost() { - return 1.0; +/** + * Estimates the cost of evaluating an always strict iterator (e.g. btree) in a non-strict context. + * + * When the estimate and strict cost is low, this models the cost of checking whether + * the seek docid matches the docid the iterator is already positioned at (the 0.2 factor). + * + * The resulting non-strict cost is most accurate when the inflow is 1.0. + * The resulting non-strict cost is highly underestimated when the inflow goes to 0.0. + * It is important to have a better estimate at higher inflows, + * as the latency (time) penalty is higher if choosing wrong. + * + * Note: This formula is equal to forced_strict_cost() in flow.h. + */ +inline double non_strict_cost_of_strict_iterator(double estimate, double strict_cost) { + return 0.2 * (1.0 - estimate) + strict_cost; } // Strict cost of matching in a btree posting list (e.g. fast-search attribute or memory index field). +// Test used: IteratorBenchmark::analyze_term_search_in_fast_search_attributes inline double btree_strict_cost(double my_est) { return my_est; } +// Non-strict cost of matching in a btree posting list (e.g. fast-search attribute or memory index field). +// Test used: IteratorBenchmark::analyze_btree_iterator_non_strict +inline double btree_cost(double my_est) { + return non_strict_cost_of_strict_iterator(my_est, btree_strict_cost(my_est)); +} + // Non-strict cost of matching in a bitvector. inline double bitvector_cost() { return 1.0; @@ -72,14 +99,16 @@ inline double bitvector_strict_cost(double my_est) { return 1.5 * my_est; } -// Non-strict cost of matching in a disk index posting list. -inline double disk_index_cost() { - return 1.5; -} - // Strict cost of matching in a disk index posting list. +// Test used: IteratorBenchmark::analyze_term_search_in_disk_index inline double disk_index_strict_cost(double my_est) { return 1.5 * my_est; } +// Non-strict cost of matching in a disk index posting list. +// Test used: IteratorBenchmark::analyze_term_search_in_disk_index +inline double disk_index_cost(double my_est) { + return non_strict_cost_of_strict_iterator(my_est, disk_index_strict_cost(my_est)); +} + } diff --git a/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp b/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp index 07e490f4575..af99260979d 100644 --- a/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp @@ -31,7 +31,6 @@ public: double calc(TypedCells rhs) const noexcept override { size_t sz = _lhs.size(); vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); - assert(sz == rhs_vector.size()); auto a = _lhs.data(); auto b = rhs_vector.data(); double b_norm_sq = _computer.dotProduct(b, b, sz); diff --git a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h index 9dbd12650cb..7ff9448c5af 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h @@ -45,7 +45,7 @@ public: if (has_single_subspace) { auto cells = _attr_tensor.get_vector(docid, 0); double min_rawscore = _dist_fun->min_rawscore(); - if (cells.size == 0) [[unlikely]] { + if ( ! cells.valid() ) [[unlikely]] { return min_rawscore; } return std::max(min_rawscore, _dist_fun->to_rawscore(_dist_fun->calc(cells))); @@ -66,7 +66,7 @@ public: double calc_with_limit(uint32_t docid, double limit) const noexcept { if (has_single_subspace) { auto cells = _attr_tensor.get_vector(docid, 0); - if (cells.size == 0) [[unlikely]] { + if ( ! cells.valid() ) [[unlikely]] { return std::numeric_limits<double>::max(); } return _dist_fun->calc_with_limit(cells, limit); diff --git a/searchlib/src/vespa/searchlib/tensor/empty_subspace.cpp b/searchlib/src/vespa/searchlib/tensor/empty_subspace.cpp index d581dbd129e..a7168b5eae6 100644 --- a/searchlib/src/vespa/searchlib/tensor/empty_subspace.cpp +++ b/searchlib/src/vespa/searchlib/tensor/empty_subspace.cpp @@ -11,7 +11,7 @@ EmptySubspace::EmptySubspace(const SubspaceType& type) { _empty_space.resize(type.mem_size()); // Set size to zero to signal empty/invalid subspace - _cells = vespalib::eval::TypedCells(&_empty_space[0], type.cell_type(), 0); + _cells = vespalib::eval::TypedCells(_empty_space.data(), type.cell_type(), 0); } EmptySubspace::~EmptySubspace() = default; diff --git a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp index 6a730132ad1..3ab3a1123eb 100644 --- a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp @@ -31,12 +31,10 @@ public: _lhs_vector(_tmpSpace.storeLhs(lhs)) {} double calc(TypedCells rhs) const noexcept override { - size_t sz = _lhs_vector.size(); vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); - assert(sz == rhs_vector.size()); auto a = _lhs_vector.data(); auto b = rhs_vector.data(); - return _computer.squaredEuclideanDistance(cast(a), cast(b), sz); + return _computer.squaredEuclideanDistance(cast(a), cast(b), _lhs_vector.size()); } double convert_threshold(double threshold) const noexcept override { return threshold*threshold; diff --git a/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp b/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp index 0be920b9c03..7f29a100492 100644 --- a/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp @@ -28,7 +28,6 @@ public: if constexpr (std::is_same<Int8Float, FloatType>::value) { return (double) vespalib::binary_hamming_distance(_lhs_vector.data(), rhs_vector.data(), sz); } else { - assert(sz == rhs_vector.size()); size_t sum = 0; for (size_t i = 0; i < sz; ++i) { sum += (_lhs_vector[i] == rhs_vector[i]) ? 0 : 1; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index bf62fcdaa23..7b42fd9a0e1 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -303,12 +303,29 @@ HnswIndex<type>::remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32 _graph.set_link_array(remove_from, level, new_links); } +namespace { + +double +calc_distance_helper(const BoundDistanceFunction &df, vespalib::eval::TypedCells rhs) +{ + if (!rhs.valid()) [[unlikely]] { + /* + * We are in a read thread and the write thread has removed the + * tensor. + */ + return std::numeric_limits<double>::max(); + } + return df.calc(rhs); +} + +} + template <HnswIndexType type> double HnswIndex<type>::calc_distance(const BoundDistanceFunction &df, uint32_t rhs_nodeid) const { auto rhs = get_vector(rhs_nodeid); - return df.calc(rhs); + return calc_distance_helper(df, rhs); } template <HnswIndexType type> @@ -316,7 +333,7 @@ double HnswIndex<type>::calc_distance(const BoundDistanceFunction &df, uint32_t rhs_docid, uint32_t rhs_subspace) const { auto rhs = get_vector(rhs_docid, rhs_subspace); - return df.calc(rhs); + return calc_distance_helper(df, rhs); } template <HnswIndexType type> diff --git a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp index 267f91bb4e0..4bc90001227 100644 --- a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp @@ -29,12 +29,10 @@ public: } } double calc(TypedCells rhs) const noexcept override { - size_t sz = _lhs.size(); vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); - assert(sz == rhs_vector.size()); auto a = _lhs.data(); auto b = rhs_vector.data(); - double dot_product = _computer.dotProduct(a, b, sz); + double dot_product = _computer.dotProduct(a, b, _lhs.size()); double distance = _lhs_norm_sq - dot_product; return distance; } diff --git a/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp index b1018555212..4753e9d7c87 100644 --- a/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp @@ -38,7 +38,6 @@ struct ConvertCellsSelector template <typename FloatType> ConstArrayRef<FloatType> TemporaryVectorStore<FloatType>::internal_convert(TypedCells cells, size_t offset) noexcept { - assert(cells.size * 2 == _tmpSpace.size()); ArrayRef<FloatType> where(_tmpSpace.data() + offset, cells.size); using MyTypify = vespalib::eval::TypifyCellType; using MySelector = ConvertCellsSelector<FloatType>; diff --git a/searchlib/src/vespa/searchlib/tensor/vector_bundle.h b/searchlib/src/vespa/searchlib/tensor/vector_bundle.h index 087c0f43b60..e8b65c5b6b2 100644 --- a/searchlib/src/vespa/searchlib/tensor/vector_bundle.h +++ b/searchlib/src/vespa/searchlib/tensor/vector_bundle.h @@ -39,7 +39,6 @@ public: ~VectorBundle() = default; uint32_t subspaces() const noexcept { return _subspaces; } vespalib::eval::TypedCells cells(uint32_t subspace) const noexcept { - assert(subspace < _subspaces); return {static_cast<const char*>(_data) + _subspace_mem_size * subspace, _cell_type, _subspace_size}; } }; diff --git a/vespajlib/src/main/java/com/yahoo/errorhandling/package-info.java b/vespajlib/src/main/java/com/yahoo/errorhandling/package-info.java deleted file mode 100644 index ac6c913381c..00000000000 --- a/vespajlib/src/main/java/com/yahoo/errorhandling/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.errorhandling; - -import com.yahoo.osgi.annotation.ExportPackage; |