diff options
67 files changed, 1149 insertions, 778 deletions
diff --git a/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java b/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java index 2f8b73839da..b7ad4ac6279 100644 --- a/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java +++ b/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java @@ -15,7 +15,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.Objects; -import java.util.OptionalLong; import java.util.function.Predicate; import java.util.zip.GZIPInputStream; @@ -26,13 +25,13 @@ import java.util.zip.GZIPInputStream; */ public class ArchiveStreamReader implements AutoCloseable { - private final ArchiveInputStream archiveInputStream; + private final ArchiveInputStream<? extends ArchiveEntry> archiveInputStream; private final Options options; private long totalRead = 0; private long entriesRead = 0; - private ArchiveStreamReader(ArchiveInputStream archiveInputStream, Options options) { + private <T extends ArchiveEntry> ArchiveStreamReader(ArchiveInputStream<T> archiveInputStream, Options options) { this.archiveInputStream = Objects.requireNonNull(archiveInputStream); this.options = Objects.requireNonNull(options); } diff --git a/client/go/go.mod b/client/go/go.mod index 0dab1722b0d..a122508b1b9 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -8,7 +8,7 @@ require ( github.com/fatih/color v1.16.0 // This is the most recent version compatible with Go 1.19. Upgrade when we upgrade our Go version github.com/go-json-experiment/json v0.0.0-20230216065249-540f01442424 - github.com/klauspost/compress v1.17.2 + github.com/klauspost/compress v1.17.3 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 diff --git a/client/go/go.sum b/client/go/go.sum index f4c69d92d9e..92e96c4e813 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -35,6 +35,8 @@ github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2g github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA= +github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/client/js/app/package.json b/client/js/app/package.json index 23ddccbe7e7..6162a6a8071 100644 --- a/client/js/app/package.json +++ b/client/js/app/package.json @@ -39,7 +39,7 @@ "pretty-quick": "^3", "react-router-dom": "^6", "use-context-selector": "^1", - "vite": "^4" + "vite": "^5.0.0" }, "jest": { "testMatch": [ diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 1f6ebac1617..bd472fbc763 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -24,9 +24,9 @@ chalk "^2.4.2" "@babel/compat-data@^7.22.9": - version "7.23.2" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.23.2.tgz#6a12ced93455827037bfb5ed8492820d60fc32cc" - integrity sha512-0S9TQMmDHlqAZ2ITT95irXKfxN9bncq8ZCoJhun3nHL/lLUxd2NKBJYoNGWH7S0hz6fRQwWlAWn/ILM0C70KZQ== + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.23.3.tgz#3febd552541e62b5e883a25eb3effd7c7379db11" + integrity sha512-BmR4bWbDIoFJmJ9z2cZ8Gmm2MXgEDgjdWgpKmKWUt54UGFJdlj31ECtbaDvCG/qVdG3AQ1SfpZEs01lUFbzLOQ== "@babel/core@^7.1.0", "@babel/core@^7.12.17": version "7.22.9" @@ -70,21 +70,21 @@ json5 "^2.2.3" semver "^6.3.1" -"@babel/core@^7.23.2": - version "7.23.2" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.2.tgz#ed10df0d580fff67c5f3ee70fd22e2e4c90a9f94" - integrity sha512-n7s51eWdaWZ3vGT2tD4T7J6eJs3QoBXydv7vkUM06Bf1cbVD2Kc2UrkzhiQwobfV7NwOnQXYL7UBJ5VPU+RGoQ== +"@babel/core@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.3.tgz#5ec09c8803b91f51cc887dedc2654a35852849c9" + integrity sha512-Jg+msLuNuCJDyBvFv5+OKOUjWMZgd85bKjbICd3zWrKAo+bJ49HJufi7CQE0q0uR8NGyO6xkCACScNqyjHSZew== dependencies: "@ampproject/remapping" "^2.2.0" "@babel/code-frame" "^7.22.13" - "@babel/generator" "^7.23.0" + "@babel/generator" "^7.23.3" "@babel/helper-compilation-targets" "^7.22.15" - "@babel/helper-module-transforms" "^7.23.0" + "@babel/helper-module-transforms" "^7.23.3" "@babel/helpers" "^7.23.2" - "@babel/parser" "^7.23.0" + "@babel/parser" "^7.23.3" "@babel/template" "^7.22.15" - "@babel/traverse" "^7.23.2" - "@babel/types" "^7.23.0" + "@babel/traverse" "^7.23.3" + "@babel/types" "^7.23.3" convert-source-map "^2.0.0" debug "^4.1.0" gensync "^1.0.0-beta.2" @@ -111,12 +111,12 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" -"@babel/generator@^7.23.0": - version "7.23.0" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" - integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== +"@babel/generator@^7.23.0", "@babel/generator@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.3.tgz#86e6e83d95903fbe7613f448613b8b319f330a8e" + integrity sha512-keeZWAV4LU3tW0qRi19HRpabC/ilM0HRBBzf9/k8FFiG4KVpiv0FIy4hHfLfFQZNhziCTPTmd59zoyv6DNISzg== dependencies: - "@babel/types" "^7.23.0" + "@babel/types" "^7.23.3" "@jridgewell/gen-mapping" "^0.3.2" "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" @@ -199,10 +199,10 @@ "@babel/helper-split-export-declaration" "^7.22.6" "@babel/helper-validator-identifier" "^7.22.5" -"@babel/helper-module-transforms@^7.23.0": - version "7.23.0" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.23.0.tgz#3ec246457f6c842c0aee62a01f60739906f7047e" - integrity sha512-WhDWw1tdrlT0gMgUJSlX0IQvoO1eN279zrAUbVB+KpV2c3Tylz8+GnKOLllCS6Z/iZQEyVYxhZVUdPTqs2YYPw== +"@babel/helper-module-transforms@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.23.3.tgz#d7d12c3c5d30af5b3c0fcab2a6d5217773e2d0f1" + integrity sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ== dependencies: "@babel/helper-environment-visitor" "^7.22.20" "@babel/helper-module-imports" "^7.22.15" @@ -280,10 +280,10 @@ chalk "^2.4.2" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.20.7", "@babel/parser@^7.22.15", "@babel/parser@^7.23.0": - version "7.23.0" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" - integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== +"@babel/parser@^7.1.0", "@babel/parser@^7.20.7", "@babel/parser@^7.22.15", "@babel/parser@^7.23.0", "@babel/parser@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.3.tgz#0ce0be31a4ca4f1884b5786057cadcb6c3be58f9" + integrity sha512-uVsWNvlVsIninV2prNz/3lHCb+5CJ+e+IUBfbjToAHODtfGYLfCFuY4AU7TskI+dAKk+njsPiBjq1gKTvZOBaw== "@babel/parser@^7.14.7", "@babel/parser@^7.22.16": version "7.22.16" @@ -402,17 +402,17 @@ "@babel/helper-plugin-utils" "^7.22.5" "@babel/helper-simple-access" "^7.22.5" -"@babel/plugin-transform-react-jsx-self@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.22.5.tgz#ca2fdc11bc20d4d46de01137318b13d04e481d8e" - integrity sha512-nTh2ogNUtxbiSbxaT4Ds6aXnXEipHweN9YRgOX/oNXdf0cCrGn/+2LozFa3lnPV5D90MkjhgckCPBrsoSc1a7g== +"@babel/plugin-transform-react-jsx-self@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.23.3.tgz#ed3e7dadde046cce761a8e3cf003a13d1a7972d9" + integrity sha512-qXRvbeKDSfwnlJnanVRp0SfuWE5DQhwQr5xtLBzp56Wabyo+4CMosF6Kfp+eOD/4FYpql64XVJ2W0pVLlJZxOQ== dependencies: "@babel/helper-plugin-utils" "^7.22.5" -"@babel/plugin-transform-react-jsx-source@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.22.5.tgz#49af1615bfdf6ed9d3e9e43e425e0b2b65d15b6c" - integrity sha512-yIiRO6yobeEIaI0RTbIr8iAK9FcBHLtZq0S89ZPjDLQXBA4xvghaKqI0etp/tF3htTM0sazJKKLz9oEiGRtu7w== +"@babel/plugin-transform-react-jsx-source@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.23.3.tgz#03527006bdc8775247a78643c51d4e715fe39a3e" + integrity sha512-91RS0MDnAWDNvGC6Wio5XYkyWI39FMFO+JK9+4AlgaTH+yWwVTsw7/sn6LK0lH7c5F+TFkpv/3LfCJ1Ydwof/g== dependencies: "@babel/helper-plugin-utils" "^7.22.5" @@ -439,7 +439,7 @@ "@babel/parser" "^7.22.15" "@babel/types" "^7.22.15" -"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.15", "@babel/traverse@^7.22.17", "@babel/traverse@^7.22.8", "@babel/traverse@^7.23.2": +"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.15", "@babel/traverse@^7.22.17", "@babel/traverse@^7.22.8": version "7.23.2" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== @@ -455,10 +455,26 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0": - version "7.23.0" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" - integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== +"@babel/traverse@^7.23.2", "@babel/traverse@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.3.tgz#26ee5f252e725aa7aca3474aa5b324eaf7908b5b" + integrity sha512-+K0yF1/9yR0oHdE0StHuEj3uTPzwwbrLGfNOndVJVV2TqA5+j3oljJUb4nmB954FLGjNem976+B+eDuLIjesiQ== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.23.3" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.3" + "@babel/types" "^7.23.3" + debug "^4.1.0" + globals "^11.1.0" + +"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.3.tgz#d5ea892c07f2ec371ac704420f4dcdb07b5f9598" + integrity sha512-OZnvoH2l8PK5eUvEcUyCt/sXgr/h+UWpVuBbOljwcrAgUl6lpchoQ++PHGyQy1AtYnVA6CEq3y5xeEI10brpXw== dependencies: "@babel/helper-string-parser" "^7.22.5" "@babel/helper-validator-identifier" "^7.22.20" @@ -583,115 +599,115 @@ resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.3.1.tgz#d0fce5d07b0620caa282b5131c297bb60f9d87e6" integrity sha512-EsBwpc7hBUJWAsNPBmJy4hxWx12v6bshQsldrVmjxJoc3isbxhOrF2IcCpaXxfvq03NwkI7sbsOLXbYuqF/8Ww== -"@esbuild/android-arm64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.18.20.tgz#984b4f9c8d0377443cc2dfcef266d02244593622" - integrity sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ== - -"@esbuild/android-arm@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.18.20.tgz#fedb265bc3a589c84cc11f810804f234947c3682" - integrity sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw== - -"@esbuild/android-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.18.20.tgz#35cf419c4cfc8babe8893d296cd990e9e9f756f2" - integrity sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg== - -"@esbuild/darwin-arm64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.18.20.tgz#08172cbeccf95fbc383399a7f39cfbddaeb0d7c1" - integrity sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA== - -"@esbuild/darwin-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.18.20.tgz#d70d5790d8bf475556b67d0f8b7c5bdff053d85d" - integrity sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ== - -"@esbuild/freebsd-arm64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.18.20.tgz#98755cd12707f93f210e2494d6a4b51b96977f54" - integrity sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw== - -"@esbuild/freebsd-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.18.20.tgz#c1eb2bff03915f87c29cece4c1a7fa1f423b066e" - integrity sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ== - -"@esbuild/linux-arm64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.18.20.tgz#bad4238bd8f4fc25b5a021280c770ab5fc3a02a0" - integrity sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA== - -"@esbuild/linux-arm@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.18.20.tgz#3e617c61f33508a27150ee417543c8ab5acc73b0" - integrity sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg== - -"@esbuild/linux-ia32@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.18.20.tgz#699391cccba9aee6019b7f9892eb99219f1570a7" - integrity sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA== - -"@esbuild/linux-loong64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.18.20.tgz#e6fccb7aac178dd2ffb9860465ac89d7f23b977d" - integrity sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg== - -"@esbuild/linux-mips64el@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.18.20.tgz#eeff3a937de9c2310de30622a957ad1bd9183231" - integrity sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ== - -"@esbuild/linux-ppc64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.18.20.tgz#2f7156bde20b01527993e6881435ad79ba9599fb" - integrity sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA== - -"@esbuild/linux-riscv64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.18.20.tgz#6628389f210123d8b4743045af8caa7d4ddfc7a6" - integrity sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A== - -"@esbuild/linux-s390x@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.18.20.tgz#255e81fb289b101026131858ab99fba63dcf0071" - integrity sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ== - -"@esbuild/linux-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.18.20.tgz#c7690b3417af318a9b6f96df3031a8865176d338" - integrity sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w== - -"@esbuild/netbsd-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.18.20.tgz#30e8cd8a3dded63975e2df2438ca109601ebe0d1" - integrity sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A== - -"@esbuild/openbsd-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.18.20.tgz#7812af31b205055874c8082ea9cf9ab0da6217ae" - integrity sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg== - -"@esbuild/sunos-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.18.20.tgz#d5c275c3b4e73c9b0ecd38d1ca62c020f887ab9d" - integrity sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ== - -"@esbuild/win32-arm64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.18.20.tgz#73bc7f5a9f8a77805f357fab97f290d0e4820ac9" - integrity sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg== - -"@esbuild/win32-ia32@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.18.20.tgz#ec93cbf0ef1085cc12e71e0d661d20569ff42102" - integrity sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g== - -"@esbuild/win32-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.18.20.tgz#786c5f41f043b07afb1af37683d7c33668858f6d" - integrity sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ== +"@esbuild/android-arm64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.19.5.tgz#276c5f99604054d3dbb733577e09adae944baa90" + integrity sha512-5d1OkoJxnYQfmC+Zd8NBFjkhyCNYwM4n9ODrycTFY6Jk1IGiZ+tjVJDDSwDt77nK+tfpGP4T50iMtVi4dEGzhQ== + +"@esbuild/android-arm@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.19.5.tgz#4a3cbf14758166abaae8ba9c01a80e68342a4eec" + integrity sha512-bhvbzWFF3CwMs5tbjf3ObfGqbl/17ict2/uwOSfr3wmxDE6VdS2GqY/FuzIPe0q0bdhj65zQsvqfArI9MY6+AA== + +"@esbuild/android-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.19.5.tgz#21a3d11cd4613d2d3c5ccb9e746c254eb9265b0a" + integrity sha512-9t+28jHGL7uBdkBjL90QFxe7DVA+KGqWlHCF8ChTKyaKO//VLuoBricQCgwhOjA1/qOczsw843Fy4cbs4H3DVA== + +"@esbuild/darwin-arm64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.19.5.tgz#714cb839f467d6a67b151ee8255886498e2b9bf6" + integrity sha512-mvXGcKqqIqyKoxq26qEDPHJuBYUA5KizJncKOAf9eJQez+L9O+KfvNFu6nl7SCZ/gFb2QPaRqqmG0doSWlgkqw== + +"@esbuild/darwin-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.19.5.tgz#2c553e97a6d2b4ae76a884e35e6cbab85a990bbf" + integrity sha512-Ly8cn6fGLNet19s0X4unjcniX24I0RqjPv+kurpXabZYSXGM4Pwpmf85WHJN3lAgB8GSth7s5A0r856S+4DyiA== + +"@esbuild/freebsd-arm64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.19.5.tgz#d554f556718adb31917a0da24277bf84b6ee87f3" + integrity sha512-GGDNnPWTmWE+DMchq1W8Sd0mUkL+APvJg3b11klSGUDvRXh70JqLAO56tubmq1s2cgpVCSKYywEiKBfju8JztQ== + +"@esbuild/freebsd-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.19.5.tgz#288f7358a3bb15d99e73c65c9adaa3dabb497432" + integrity sha512-1CCwDHnSSoA0HNwdfoNY0jLfJpd7ygaLAp5EHFos3VWJCRX9DMwWODf96s9TSse39Br7oOTLryRVmBoFwXbuuQ== + +"@esbuild/linux-arm64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.19.5.tgz#95933ae86325c93cb6b5e8333d22120ecfdc901b" + integrity sha512-o3vYippBmSrjjQUCEEiTZ2l+4yC0pVJD/Dl57WfPwwlvFkrxoSO7rmBZFii6kQB3Wrn/6GwJUPLU5t52eq2meA== + +"@esbuild/linux-arm@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.19.5.tgz#0acef93aa3e0579e46d33b666627bddb06636664" + integrity sha512-lrWXLY/vJBzCPC51QN0HM71uWgIEpGSjSZZADQhq7DKhPcI6NH1IdzjfHkDQws2oNpJKpR13kv7/pFHBbDQDwQ== + +"@esbuild/linux-ia32@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.19.5.tgz#b6e5c9e80b42131cbd6b1ddaa48c92835f1ed67f" + integrity sha512-MkjHXS03AXAkNp1KKkhSKPOCYztRtK+KXDNkBa6P78F8Bw0ynknCSClO/ztGszILZtyO/lVKpa7MolbBZ6oJtQ== + +"@esbuild/linux-loong64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.19.5.tgz#e5f0cf95a180158b01ff5f417da796a1c09dfbea" + integrity sha512-42GwZMm5oYOD/JHqHska3Jg0r+XFb/fdZRX+WjADm3nLWLcIsN27YKtqxzQmGNJgu0AyXg4HtcSK9HuOk3v1Dw== + +"@esbuild/linux-mips64el@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.19.5.tgz#ae36fb86c7d5f641f3a0c8472e83dcb6ea36a408" + integrity sha512-kcjndCSMitUuPJobWCnwQ9lLjiLZUR3QLQmlgaBfMX23UEa7ZOrtufnRds+6WZtIS9HdTXqND4yH8NLoVVIkcg== + +"@esbuild/linux-ppc64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.19.5.tgz#7960cb1666f0340ddd9eef7b26dcea3835d472d0" + integrity sha512-yJAxJfHVm0ZbsiljbtFFP1BQKLc8kUF6+17tjQ78QjqjAQDnhULWiTA6u0FCDmYT1oOKS9PzZ2z0aBI+Mcyj7Q== + +"@esbuild/linux-riscv64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.19.5.tgz#32207df26af60a3a9feea1783fc21b9817bade19" + integrity sha512-5u8cIR/t3gaD6ad3wNt1MNRstAZO+aNyBxu2We8X31bA8XUNyamTVQwLDA1SLoPCUehNCymhBhK3Qim1433Zag== + +"@esbuild/linux-s390x@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.19.5.tgz#b38d5681db89a3723862dfa792812397b1510a7d" + integrity sha512-Z6JrMyEw/EmZBD/OFEFpb+gao9xJ59ATsoTNlj39jVBbXqoZm4Xntu6wVmGPB/OATi1uk/DB+yeDPv2E8PqZGw== + +"@esbuild/linux-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.19.5.tgz#46feba2ad041a241379d150f415b472fe3885075" + integrity sha512-psagl+2RlK1z8zWZOmVdImisMtrUxvwereIdyJTmtmHahJTKb64pAcqoPlx6CewPdvGvUKe2Jw+0Z/0qhSbG1A== + +"@esbuild/netbsd-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.19.5.tgz#3b5c1fb068f26bfc681d31f682adf1bea4ef0702" + integrity sha512-kL2l+xScnAy/E/3119OggX8SrWyBEcqAh8aOY1gr4gPvw76la2GlD4Ymf832UCVbmuWeTf2adkZDK+h0Z/fB4g== + +"@esbuild/openbsd-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.19.5.tgz#ca6830316ca68056c5c88a875f103ad3235e00db" + integrity sha512-sPOfhtzFufQfTBgRnE1DIJjzsXukKSvZxloZbkJDG383q0awVAq600pc1nfqBcl0ice/WN9p4qLc39WhBShRTA== + +"@esbuild/sunos-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.19.5.tgz#9efc4eb9539a7be7d5a05ada52ee43cda0d8e2dd" + integrity sha512-dGZkBXaafuKLpDSjKcB0ax0FL36YXCvJNnztjKV+6CO82tTYVDSH2lifitJ29jxRMoUhgkg9a+VA/B03WK5lcg== + +"@esbuild/win32-arm64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.19.5.tgz#29f8184afa7a02a956ebda4ed638099f4b8ff198" + integrity sha512-dWVjD9y03ilhdRQ6Xig1NWNgfLtf2o/STKTS+eZuF90fI2BhbwD6WlaiCGKptlqXlURVB5AUOxUj09LuwKGDTg== + +"@esbuild/win32-ia32@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.19.5.tgz#f3de07afb292ecad651ae4bb8727789de2d95b05" + integrity sha512-4liggWIA4oDgUxqpZwrDhmEfAH4d0iljanDOK7AnVU89T6CzHon/ony8C5LeOdfgx60x5cnQJFZwEydVlYx4iw== + +"@esbuild/win32-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.19.5.tgz#faad84c41ba12e3a0acb52571df9bff37bee75f6" + integrity sha512-czTrygUsB/jlM8qEW5MD8bgYU2Xg14lo6kBDXW6HdxKjh8M5PzETGiSHaz9MtbXBYDloHNUAUW2tMiKW4KM9Mw== "@eslint-community/eslint-utils@^4.2.0": version "4.4.0" @@ -1254,10 +1270,70 @@ dependencies: "@babel/runtime" "^7.13.10" -"@remix-run/router@1.11.0": - version "1.11.0" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.11.0.tgz#e0e45ac3fff9d8a126916f166809825537e9f955" - integrity sha512-BHdhcWgeiudl91HvVa2wxqZjSHbheSgIiDvxrF1VjFzBzpTtuDPkOdOi3Iqvc08kXtFkLjhbS+ML9aM8mJS+wQ== +"@remix-run/router@1.12.0": + version "1.12.0" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.12.0.tgz#e89b64b6fa97a8a5b740a4c38c2904b80f1f229a" + integrity sha512-2hXv036Bux90e1GXTWSMfNzfDDK8LA8JYEWfyHxzvwdp6GyoWEovKc9cotb3KCKmkdwsIBuFGX7ScTWyiHv7Eg== + +"@rollup/rollup-android-arm-eabi@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.4.1.tgz#f276b0fa322270aa42d1f56c982db6ef8d6a4393" + integrity sha512-Ss4suS/sd+6xLRu+MLCkED2mUrAyqHmmvZB+zpzZ9Znn9S8wCkTQCJaQ8P8aHofnvG5L16u9MVnJjCqioPErwQ== + +"@rollup/rollup-android-arm64@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.4.1.tgz#f0492f00d18e1067785f8e820e137c00528c5e62" + integrity sha512-sRSkGTvGsARwWd7TzC8LKRf8FiPn7257vd/edzmvG4RIr9x68KBN0/Ek48CkuUJ5Pj/Dp9vKWv6PEupjKWjTYA== + +"@rollup/rollup-darwin-arm64@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.4.1.tgz#40443db7f4559171d797581e0618ec1a4c8dcee9" + integrity sha512-nz0AiGrrXyaWpsmBXUGOBiRDU0wyfSXbFuF98pPvIO8O6auQsPG6riWsfQqmCCC5FNd8zKQ4JhgugRNAkBJ8mQ== + +"@rollup/rollup-darwin-x64@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.4.1.tgz#2868f37a9f9c2c22c091b6209f6ce7454437edf9" + integrity sha512-Ogqvf4/Ve/faMaiPRvzsJEqajbqs00LO+8vtrPBVvLgdw4wBg6ZDXdkDAZO+4MLnrc8mhGV6VJAzYScZdPLtJg== + +"@rollup/rollup-linux-arm-gnueabihf@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.4.1.tgz#d78d7ad358d24058166ab5599de3dcb5ab951add" + integrity sha512-9zc2tqlr6HfO+hx9+wktUlWTRdje7Ub15iJqKcqg5uJZ+iKqmd2CMxlgPpXi7+bU7bjfDIuvCvnGk7wewFEhCg== + +"@rollup/rollup-linux-arm64-gnu@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.4.1.tgz#5d07588b40a04f5b6fbd9e0169c8dc32c1c2ed21" + integrity sha512-phLb1fN3rq2o1j1v+nKxXUTSJnAhzhU0hLrl7Qzb0fLpwkGMHDem+o6d+ZI8+/BlTXfMU4kVWGvy6g9k/B8L6Q== + +"@rollup/rollup-linux-arm64-musl@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.4.1.tgz#d452e88a02755f449f6e98d4ce424d655ef42cfe" + integrity sha512-M2sDtw4tf57VPSjbTAN/lz1doWUqO2CbQuX3L9K6GWIR5uw9j+ROKCvvUNBY8WUbMxwaoc8mH9HmmBKsLht7+w== + +"@rollup/rollup-linux-x64-gnu@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.4.1.tgz#e8e8e87ab098784383a5ced4aa4bbfa7b2c92a4e" + integrity sha512-mHIlRLX+hx+30cD6c4BaBOsSqdnCE4ok7/KDvjHYAHoSuveoMMxIisZFvcLhUnyZcPBXDGZTuBoalcuh43UfQQ== + +"@rollup/rollup-linux-x64-musl@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.4.1.tgz#3e5da42626672e2d620ed12746158b0cf6143b23" + integrity sha512-tB+RZuDi3zxFx7vDrjTNGVLu2KNyzYv+UY8jz7e4TMEoAj7iEt8Qk6xVu6mo3pgjnsHj6jnq3uuRsHp97DLwOA== + +"@rollup/rollup-win32-arm64-msvc@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.4.1.tgz#0f0d0c6b75c53643fab8238c76889a95bca3b9cc" + integrity sha512-Hdn39PzOQowK/HZzYpCuZdJC91PE6EaGbTe2VCA9oq2u18evkisQfws0Smh9QQGNNRa/T7MOuGNQoLeXhhE3PQ== + +"@rollup/rollup-win32-ia32-msvc@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.4.1.tgz#8bb9e8fbf0fdf96fe3bebcee23f5cfdbbd9a4a0a" + integrity sha512-tLpKb1Elm9fM8c5w3nl4N1eLTP4bCqTYw9tqUBxX8/hsxqHO3dxc2qPbZ9PNkdK4tg4iLEYn0pOUnVByRd2CbA== + +"@rollup/rollup-win32-x64-msvc@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.4.1.tgz#8311b77e6cce322865ba12ada8c3779369610d18" + integrity sha512-eAhItDX9yQtZVM3yvXS/VR3qPqcnXvnLyx1pLXl4JzyNMBNO3KC986t/iAg2zcMzpAp9JSvxB5VZGnBiNoA98w== "@sinclair/typebox@^0.27.8": version "0.27.8" @@ -1289,10 +1365,10 @@ "@types/babel__template" "*" "@types/babel__traverse" "*" -"@types/babel__core@^7.20.3": - version "7.20.3" - resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.3.tgz#d5625a50b6f18244425a1359a858c73d70340778" - integrity sha512-54fjTSeSHwfan8AyHWrKbfBWiEUrNTZsUwPTDSNaaP1QDQIZbeNUg3a59E9D+375MzUw/x1vx2/0F5LBz+AeYA== +"@types/babel__core@^7.20.4": + version "7.20.4" + resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.4.tgz#26a87347e6c6f753b3668398e34496d6d9ac6ac0" + integrity sha512-mLnSC22IC4vcWiuObSRjrLd9XcBTGf59vUSoq2jkQDJ/QQ8PMI9rSuzE+aEV8karUMbskw07bKYoUJCKTUaygg== dependencies: "@babel/parser" "^7.20.7" "@babel/types" "^7.20.7" @@ -1301,24 +1377,24 @@ "@types/babel__traverse" "*" "@types/babel__generator@*": - version "7.6.6" - resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.6.tgz#676f89f67dc8ddaae923f70ebc5f1fa800c031a8" - integrity sha512-66BXMKb/sUWbMdBNdMvajU7i/44RkrA3z/Yt1c7R5xejt8qh84iU54yUWCtm0QwGJlDcf/gg4zd/x4mpLAlb/w== + version "7.6.7" + resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.7.tgz#a7aebf15c7bc0eb9abd638bdb5c0b8700399c9d0" + integrity sha512-6Sfsq+EaaLrw4RmdFWE9Onp63TOUue71AWb4Gpa6JxzgTYtimbM086WnYTy2U67AofR++QKCo08ZP6pwx8YFHQ== dependencies: "@babel/types" "^7.0.0" "@types/babel__template@*": - version "7.4.3" - resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.3.tgz#db9ac539a2fe05cfe9e168b24f360701bde41f5f" - integrity sha512-ciwyCLeuRfxboZ4isgdNZi/tkt06m8Tw6uGbBSBgWrnnZGNXiEyM27xc/PjXGQLqlZ6ylbgHMnm7ccF9tCkOeQ== + version "7.4.4" + resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.4.tgz#5672513701c1b2199bc6dad636a9d7491586766f" + integrity sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A== dependencies: "@babel/parser" "^7.1.0" "@babel/types" "^7.0.0" "@types/babel__traverse@*": - version "7.20.3" - resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.3.tgz#a971aa47441b28ef17884ff945d0551265a2d058" - integrity sha512-Lsh766rGEFbaxMIDH7Qa+Yha8cMVI3qAK6CHt3OR0YfxOIn5Z54iHiyDRycHrBqeIiqGa20Kpsv1cavfBKkRSw== + version "7.20.4" + resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.4.tgz#ec2c06fed6549df8bc0eb4615b683749a4a92e1b" + integrity sha512-mSM/iKUk5fDDrEV/e83qY+Cr3I1+Q3qqTuEn++HAWYjEa1+NxZr6CNrcJGf2ZTnq4HoFGC3zaTPZTobCzCFukA== dependencies: "@babel/types" "^7.20.7" @@ -1405,14 +1481,14 @@ integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== "@vitejs/plugin-react@^4": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.1.1.tgz#a10254dc76778027407d01b6ddbca53b23852a72" - integrity sha512-Jie2HERK+uh27e+ORXXwEP5h0Y2lS9T2PRGbfebiHGlwzDO0dEnd2aNtOR/qjBlPb1YgxwAONeblL1xqLikLag== + version "4.2.0" + resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.2.0.tgz#d71352b1a443c09c7aae8f278dd071ab3d9d8490" + integrity sha512-+MHTH/e6H12kRp5HUkzOGqPMksezRMmW+TNzlh/QXfI8rRf6l2Z2yH/v12no1UvTwhZgEDMuQ7g7rrfMseU6FQ== dependencies: - "@babel/core" "^7.23.2" - "@babel/plugin-transform-react-jsx-self" "^7.22.5" - "@babel/plugin-transform-react-jsx-source" "^7.22.5" - "@types/babel__core" "^7.20.3" + "@babel/core" "^7.23.3" + "@babel/plugin-transform-react-jsx-self" "^7.23.3" + "@babel/plugin-transform-react-jsx-source" "^7.23.3" + "@types/babel__core" "^7.20.4" react-refresh "^0.14.0" acorn-jsx@^5.3.2: @@ -1894,9 +1970,9 @@ camelcase@^6.2.0: integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== caniuse-lite@^1.0.30001541: - version "1.0.30001559" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001559.tgz#95a982440d3d314c471db68d02664fb7536c5a30" - integrity sha512-cPiMKZgqgkg5LY3/ntGeLFUpi6tzddBNS58A4tnTgQw1zON7u2sZMU7SzOeVH4tj20++9ggL+V6FDOFMTaFFYA== + version "1.0.30001562" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001562.tgz#9d16c5fd7e9c592c4cd5e304bc0f75b0008b2759" + integrity sha512-kfte3Hym//51EdX4239i+Rmp20EsLIYGdPkERegTgU19hQWCRhsRFGKHTliUlsry53tv17K7n077Kqa0WJU4ng== capture-exit@^2.0.0: version "2.0.0" @@ -2237,9 +2313,9 @@ dom-helpers@^5.0.1: csstype "^3.0.2" electron-to-chromium@^1.4.535: - version "1.4.574" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.574.tgz#6de04d7c6e244e5ffcae76d2e2a33b02cab66781" - integrity sha512-bg1m8L0n02xRzx4LsTTMbBPiUd9yIR+74iPtS/Ao65CuXvhVZHP0ym1kSdDG3yHFDXqHQQBKujlN1AQ8qZnyFg== + version "1.4.586" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.586.tgz#68683163ed52a111213e2482ff847e76a5c6e891" + integrity sha512-qMa+E6yf1fNQbg3G66pHLXeJUP5CCCzNat1VPczOZOqgI2w4u+8y9sQnswMdGs5m4C1rOePq37EVBr/nsPQY7w== emittery@^0.13.1: version "0.13.1" @@ -2364,33 +2440,33 @@ esbuild-jest@^0: "@babel/plugin-transform-modules-commonjs" "^7.12.13" babel-jest "^26.6.3" -esbuild@^0.18.10: - version "0.18.20" - resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.18.20.tgz#4709f5a34801b43b799ab7d6d82f7284a9b7a7a6" - integrity sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA== +esbuild@^0.19.3: + version "0.19.5" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.19.5.tgz#53a0e19dfbf61ba6c827d51a80813cf071239a8c" + integrity sha512-bUxalY7b1g8vNhQKdB24QDmHeY4V4tw/s6Ak5z+jJX9laP5MoQseTOMemAr0gxssjNcH0MCViG8ONI2kksvfFQ== optionalDependencies: - "@esbuild/android-arm" "0.18.20" - "@esbuild/android-arm64" "0.18.20" - "@esbuild/android-x64" "0.18.20" - "@esbuild/darwin-arm64" "0.18.20" - "@esbuild/darwin-x64" "0.18.20" - "@esbuild/freebsd-arm64" "0.18.20" - "@esbuild/freebsd-x64" "0.18.20" - "@esbuild/linux-arm" "0.18.20" - "@esbuild/linux-arm64" "0.18.20" - "@esbuild/linux-ia32" "0.18.20" - "@esbuild/linux-loong64" "0.18.20" - "@esbuild/linux-mips64el" "0.18.20" - "@esbuild/linux-ppc64" "0.18.20" - "@esbuild/linux-riscv64" "0.18.20" - "@esbuild/linux-s390x" "0.18.20" - "@esbuild/linux-x64" "0.18.20" - "@esbuild/netbsd-x64" "0.18.20" - "@esbuild/openbsd-x64" "0.18.20" - "@esbuild/sunos-x64" "0.18.20" - "@esbuild/win32-arm64" "0.18.20" - "@esbuild/win32-ia32" "0.18.20" - "@esbuild/win32-x64" "0.18.20" + "@esbuild/android-arm" "0.19.5" + "@esbuild/android-arm64" "0.19.5" + "@esbuild/android-x64" "0.19.5" + "@esbuild/darwin-arm64" "0.19.5" + "@esbuild/darwin-x64" "0.19.5" + "@esbuild/freebsd-arm64" "0.19.5" + "@esbuild/freebsd-x64" "0.19.5" + "@esbuild/linux-arm" "0.19.5" + "@esbuild/linux-arm64" "0.19.5" + "@esbuild/linux-ia32" "0.19.5" + "@esbuild/linux-loong64" "0.19.5" + "@esbuild/linux-mips64el" "0.19.5" + "@esbuild/linux-ppc64" "0.19.5" + "@esbuild/linux-riscv64" "0.19.5" + "@esbuild/linux-s390x" "0.19.5" + "@esbuild/linux-x64" "0.19.5" + "@esbuild/netbsd-x64" "0.19.5" + "@esbuild/openbsd-x64" "0.19.5" + "@esbuild/sunos-x64" "0.19.5" + "@esbuild/win32-arm64" "0.19.5" + "@esbuild/win32-ia32" "0.19.5" + "@esbuild/win32-x64" "0.19.5" escalade@^3.1.1: version "3.1.1" @@ -2852,7 +2928,7 @@ fsevents@^2.1.2: resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== -fsevents@^2.3.2, fsevents@~2.3.2: +fsevents@^2.3.2, fsevents@~2.3.2, fsevents@~2.3.3: version "2.3.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== @@ -4634,7 +4710,7 @@ posix-character-classes@^0.1.0: resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab" integrity sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg== -postcss@^8.4.27: +postcss@^8.4.31: version "8.4.31" resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.31.tgz#92b451050a9f914da6755af352bdc0192508656d" integrity sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ== @@ -4764,19 +4840,19 @@ react-remove-scroll@^2.5.5: use-sidecar "^1.1.2" react-router-dom@^6: - version "6.18.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.18.0.tgz#0a50c167209d6e7bd2ed9de200a6579ea4fb1dca" - integrity sha512-Ubrue4+Ercc/BoDkFQfc6og5zRQ4A8YxSO3Knsne+eRbZ+IepAsK249XBH/XaFuOYOYr3L3r13CXTLvYt5JDjw== + version "6.19.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.19.0.tgz#ee807e36ae7dd954db7a3f770e38b7cc026c66a8" + integrity sha512-N6dWlcgL2w0U5HZUUqU2wlmOrSb3ighJmtQ438SWbhB1yuLTXQ8yyTBMK3BSvVjp7gBtKurT554nCtMOgxCZmQ== dependencies: - "@remix-run/router" "1.11.0" - react-router "6.18.0" + "@remix-run/router" "1.12.0" + react-router "6.19.0" -react-router@6.18.0: - version "6.18.0" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.18.0.tgz#32e2bedc318e095a48763b5ed7758e54034cd36a" - integrity sha512-vk2y7Dsy8wI02eRRaRmOs9g2o+aE72YCx5q9VasT1N9v+lrdB79tIqrjMfByHiY5+6aYkH2rUa5X839nwWGPDg== +react-router@6.19.0: + version "6.19.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.19.0.tgz#6d5062fa33495859daca98d86292ab03b037a9ca" + integrity sha512-0W63PKCZ7+OuQd7Tm+RbkI8kCLmn4GPjDbX61tWljPxWgqTKlEpeQUwPkT1DRjYhF8KSihK0hQpmhU4uxVMcdw== dependencies: - "@remix-run/router" "1.11.0" + "@remix-run/router" "1.12.0" react-style-singleton@^2.2.1: version "2.2.1" @@ -4961,11 +5037,23 @@ rimraf@^3.0.2: dependencies: glob "^7.1.3" -rollup@^3.27.1: - version "3.29.4" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-3.29.4.tgz#4d70c0f9834146df8705bfb69a9a19c9e1109981" - integrity sha512-oWzmBZwvYrU0iJHtDmhsm662rC15FRXmcjCk1xD771dFDx5jJ02ufAQQTn0etB2emNk4J9EZg/yWKpsn9BWGRw== +rollup@^4.2.0: + version "4.4.1" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.4.1.tgz#2f85169f23d13dabb3d9b846d753965757353820" + integrity sha512-idZzrUpWSblPJX66i+GzrpjKE3vbYrlWirUHteoAbjKReZwa0cohAErOYA5efoMmNCdvG9yrJS+w9Kl6csaH4w== optionalDependencies: + "@rollup/rollup-android-arm-eabi" "4.4.1" + "@rollup/rollup-android-arm64" "4.4.1" + "@rollup/rollup-darwin-arm64" "4.4.1" + "@rollup/rollup-darwin-x64" "4.4.1" + "@rollup/rollup-linux-arm-gnueabihf" "4.4.1" + "@rollup/rollup-linux-arm64-gnu" "4.4.1" + "@rollup/rollup-linux-arm64-musl" "4.4.1" + "@rollup/rollup-linux-x64-gnu" "4.4.1" + "@rollup/rollup-linux-x64-musl" "4.4.1" + "@rollup/rollup-win32-arm64-msvc" "4.4.1" + "@rollup/rollup-win32-ia32-msvc" "4.4.1" + "@rollup/rollup-win32-x64-msvc" "4.4.1" fsevents "~2.3.2" rsvp@^4.8.4: @@ -5612,16 +5700,16 @@ v8-to-istanbul@^9.0.1: "@types/istanbul-lib-coverage" "^2.0.1" convert-source-map "^1.6.0" -vite@^4: - version "4.5.0" - resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.0.tgz#ec406295b4167ac3bc23e26f9c8ff559287cff26" - integrity sha512-ulr8rNLA6rkyFAlVWw2q5YJ91v098AFQ2R0PRFwPzREXOUJQPtFUG0t+/ZikhaOCDqFoDhN6/v8Sq0o4araFAw== +vite@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/vite/-/vite-5.0.0.tgz#3bfb65acda2a97127e4fa240156664a1f234ce08" + integrity sha512-ESJVM59mdyGpsiNAeHQOR/0fqNoOyWPYesFto8FFZugfmhdHx8Fzd8sF3Q/xkVhZsyOxHfdM7ieiVAorI9RjFw== dependencies: - esbuild "^0.18.10" - postcss "^8.4.27" - rollup "^3.27.1" + esbuild "^0.19.3" + postcss "^8.4.31" + rollup "^4.2.0" optionalDependencies: - fsevents "~2.3.2" + fsevents "~2.3.3" walker@^1.0.7, walker@^1.0.8, walker@~1.0.5: version "1.0.8" diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 919bce460f2..1b505cdbfae 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -1485,7 +1485,6 @@ "abstract" ], "methods" : [ - "public com.yahoo.config.model.api.OnnxModelCost$Calculator newCalculator(com.yahoo.config.application.api.ApplicationPackage, com.yahoo.config.application.api.DeployLogger)", "public abstract com.yahoo.config.model.api.OnnxModelCost$Calculator newCalculator(com.yahoo.config.application.api.ApplicationPackage, com.yahoo.config.provision.ApplicationId)", "public static com.yahoo.config.model.api.OnnxModelCost disabled()" ], diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java index abfddfe40be..acb88070482 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java @@ -1,10 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.config.model.api; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; import java.net.URI; @@ -14,11 +12,6 @@ import java.net.URI; */ public interface OnnxModelCost { - // TODO: Remove when 8.250 is oldest model in use - default Calculator newCalculator(ApplicationPackage appPkg, DeployLogger deployLogger) { - return newCalculator(appPkg, ApplicationId.defaultId()); - } - Calculator newCalculator(ApplicationPackage appPkg, ApplicationId applicationId); interface Calculator { diff --git a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java index f3f09150c1d..867ffdb3960 100644 --- a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java +++ b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java @@ -3,6 +3,7 @@ package com.yahoo.schema; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.model.container.component.OnnxModelOptions; import com.yahoo.vespa.model.ml.OnnxModelInfo; import java.util.Collections; @@ -27,10 +28,7 @@ public class OnnxModel extends DistributableResource implements Cloneable { private final Set<String> initializers = new HashSet<>(); // Runtime options - private String statelessExecutionMode = null; - private Integer statelessInterOpThreads = null; - private Integer statelessIntraOpThreads = null; - private GpuDevice gpuDevice = null; + private OnnxModelOptions onnxModelOptions = OnnxModelOptions.empty(); public OnnxModel(String name) { super(name); @@ -133,50 +131,44 @@ public class OnnxModel extends DistributableResource implements Cloneable { public void setStatelessExecutionMode(String executionMode) { if ("parallel".equalsIgnoreCase(executionMode)) { - this.statelessExecutionMode = "parallel"; + onnxModelOptions = onnxModelOptions.withExecutionMode("parallel"); } else if ("sequential".equalsIgnoreCase(executionMode)) { - this.statelessExecutionMode = "sequential"; + onnxModelOptions = onnxModelOptions.withExecutionMode("sequential"); } } public Optional<String> getStatelessExecutionMode() { - return Optional.ofNullable(statelessExecutionMode); + return onnxModelOptions.executionMode(); } public void setStatelessInterOpThreads(int interOpThreads) { if (interOpThreads >= 0) { - this.statelessInterOpThreads = interOpThreads; + onnxModelOptions = onnxModelOptions.withInterOpThreads(interOpThreads); } } public Optional<Integer> getStatelessInterOpThreads() { - return Optional.ofNullable(statelessInterOpThreads); + return onnxModelOptions.interOpThreads(); } public void setStatelessIntraOpThreads(int intraOpThreads) { if (intraOpThreads >= 0) { - this.statelessIntraOpThreads = intraOpThreads; + onnxModelOptions = onnxModelOptions.withIntraOpThreads(intraOpThreads); } } public Optional<Integer> getStatelessIntraOpThreads() { - return Optional.ofNullable(statelessIntraOpThreads); + return onnxModelOptions.intraOpThreads(); } public void setGpuDevice(int deviceNumber, boolean required) { if (deviceNumber >= 0) { - this.gpuDevice = new GpuDevice(deviceNumber, required); + onnxModelOptions = onnxModelOptions.withGpuDevice(new OnnxModelOptions.GpuDevice(deviceNumber, required)); } } - public Optional<GpuDevice> getGpuDevice() { - return Optional.ofNullable(gpuDevice); - } - - public record GpuDevice(int deviceNumber, boolean required) { - public GpuDevice { - if (deviceNumber < 0) throw new IllegalArgumentException("deviceNumber cannot be negative, got " + deviceNumber); - } + public Optional<OnnxModelOptions.GpuDevice> getGpuDevice() { + return onnxModelOptions.gpuDevice(); } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index 87b79ddcdc3..8606599f530 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -186,6 +186,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private RankingExpression secondPhaseRanking; private RankingExpression globalPhaseRanking; private final int globalPhaseRerankCount; + private final SerializationContext functionSerializationContext; /** * Creates a raw rank profile from the given rank profile @@ -225,7 +226,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { List<ExpressionFunction> functionExpressions = functions.values().stream().map(RankProfile.RankingExpressionFunction::function).toList(); Map<String, String> functionProperties = new LinkedHashMap<>(); var typeContext = compiled.typeContext(queryProfiles); - SerializationContext functionSerializationContext = new SerializationContext(functionExpressions, Map.of(), typeContext); + this.functionSerializationContext = new SerializationContext(functionExpressions, Map.of(), typeContext); if (firstPhaseRanking != null) { functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext)); } @@ -265,8 +266,6 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private void derivePropertiesAndFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions, Map<String, String> functionProperties, SerializationContext functionContext) { - if (functions.isEmpty()) return; - replaceFunctionFeatures(summaryFeatures, functionContext); replaceFunctionFeatures(matchFeatures, functionContext); @@ -556,11 +555,12 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if ("".equals(name)) name = phase; + String expressionAsString = expression.getRoot().toString(functionSerializationContext).toString(); if (expression.getRoot() instanceof ReferenceNode) { - properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); + properties.add(new Pair<>("vespa.rank." + phase, expressionAsString)); } else { properties.add(new Pair<>("vespa.rank." + phase, wrapInRankingExpression(name))); - properties.add(new Pair<>(RankingExpression.propertyName(name), expression.getRoot().toString())); + properties.add(new Pair<>(RankingExpression.propertyName(name), expressionAsString)); } return properties; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java index a644382625b..ea3caadc23a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java @@ -1,5 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; @@ -8,6 +7,8 @@ import com.yahoo.embedding.BertBaseEmbedderConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.w3c.dom.Element; +import static com.yahoo.embedding.BertBaseEmbedderConfig.OnnxExecutionMode; +import static com.yahoo.embedding.BertBaseEmbedderConfig.PoolingStrategy; import static com.yahoo.text.XML.getChildValue; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; @@ -16,6 +17,7 @@ import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATI */ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConfig.Producer { + private final OnnxModelOptions onnxModelOptions; private final ModelReference modelRef; private final ModelReference vocabRef; private final Integer maxTokens; @@ -23,18 +25,18 @@ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConf private final String transformerAttentionMask; private final String transformerTokenTypeIds; private final String transformerOutput; - private final Integer tranformerStartSequenceToken; + private final Integer transformerStartSequenceToken; private final Integer transformerEndSequenceToken; private final String poolingStrategy; - private final String onnxExecutionMode; - private final Integer onnxInteropThreads; - private final Integer onnxIntraopThreads; - private final Integer onnxGpuDevice; - public BertEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.BertBaseEmbedder", INTEGRATION_BUNDLE_NAME, xml); var model = Model.fromXml(state, xml, "transformer-model").orElseThrow(); + this.onnxModelOptions = new OnnxModelOptions( + getChildValue(xml, "onnx-execution-mode"), + getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).map(OnnxModelOptions.GpuDevice::new)); modelRef = model.modelReference(); vocabRef = Model.fromXml(state, xml, "tokenizer-vocab").orElseThrow().modelReference(); maxTokens = getChildValue(xml, "max-tokens").map(Integer::parseInt).orElse(null); @@ -42,13 +44,9 @@ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConf transformerAttentionMask = getChildValue(xml, "transformer-attention-mask").orElse(null); transformerTokenTypeIds = getChildValue(xml, "transformer-token-type-ids").orElse(null); transformerOutput = getChildValue(xml, "transformer-output").orElse(null); - tranformerStartSequenceToken = getChildValue(xml, "transformer-start-sequence-token").map(Integer::parseInt).orElse(null); + transformerStartSequenceToken = getChildValue(xml, "transformer-start-sequence-token").map(Integer::parseInt).orElse(null); transformerEndSequenceToken = getChildValue(xml, "transformer-end-sequence-token").map(Integer::parseInt).orElse(null); poolingStrategy = getChildValue(xml, "pooling-strategy").orElse(null); - onnxExecutionMode = getChildValue(xml, "onnx-execution-mode").orElse(null); - onnxInteropThreads = getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt).orElse(null); - onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); - onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); model.registerOnnxModelCost(cluster); } @@ -60,12 +58,13 @@ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConf if (transformerAttentionMask != null) b.transformerAttentionMask(transformerAttentionMask); if (transformerTokenTypeIds != null) b.transformerTokenTypeIds(transformerTokenTypeIds); if (transformerOutput != null) b.transformerOutput(transformerOutput); - if (tranformerStartSequenceToken != null) b.transformerStartSequenceToken(tranformerStartSequenceToken); + if (transformerStartSequenceToken != null) b.transformerStartSequenceToken(transformerStartSequenceToken); if (transformerEndSequenceToken != null) b.transformerEndSequenceToken(transformerEndSequenceToken); - if (poolingStrategy != null) b.poolingStrategy(BertBaseEmbedderConfig.PoolingStrategy.Enum.valueOf(poolingStrategy)); - if (onnxExecutionMode != null) b.onnxExecutionMode(BertBaseEmbedderConfig.OnnxExecutionMode.Enum.valueOf(onnxExecutionMode)); - if (onnxInteropThreads != null) b.onnxInterOpThreads(onnxInteropThreads); - if (onnxIntraopThreads != null) b.onnxIntraOpThreads(onnxIntraopThreads); - if (onnxGpuDevice != null) b.onnxGpuDevice(onnxGpuDevice); + if (poolingStrategy != null) b.poolingStrategy(PoolingStrategy.Enum.valueOf(poolingStrategy)); + onnxModelOptions.executionMode().ifPresent(value -> b.onnxExecutionMode(OnnxExecutionMode.Enum.valueOf(value))); + onnxModelOptions.interOpThreads().ifPresent(b::onnxInterOpThreads); + onnxModelOptions.intraOpThreads().ifPresent(b::onnxIntraOpThreads); + onnxModelOptions.gpuDevice().ifPresent(value -> b.onnxGpuDevice(value.deviceNumber())); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java index ed56579988d..cbae50b400c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java @@ -1,5 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; @@ -8,6 +7,7 @@ import com.yahoo.embedding.ColBertEmbedderConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.w3c.dom.Element; +import static com.yahoo.embedding.ColBertEmbedderConfig.TransformerExecutionMode; import static com.yahoo.text.XML.getChildValue; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; @@ -16,6 +16,8 @@ import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATI * @author bergum */ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderConfig.Producer { + + private final OnnxModelOptions onnxModelOptions; private final ModelReference modelRef; private final ModelReference vocabRef; @@ -31,14 +33,15 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo private final String transformerAttentionMask; private final String transformerOutput; - private final String onnxExecutionMode; - private final Integer onnxInteropThreads; - private final Integer onnxIntraopThreads; - private final Integer onnxGpuDevice; public ColBertEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.ColBertEmbedder", INTEGRATION_BUNDLE_NAME, xml); var model = Model.fromXml(state, xml, "transformer-model").orElseThrow(); + this.onnxModelOptions = new OnnxModelOptions( + getChildValue(xml, "onnx-execution-mode"), + getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).map(OnnxModelOptions.GpuDevice::new)); modelRef = model.modelReference(); vocabRef = Model.fromXml(state, xml, "tokenizer-model") .map(Model::modelReference) @@ -52,10 +55,6 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo transformerInputIds = getChildValue(xml, "transformer-input-ids").orElse(null); transformerAttentionMask = getChildValue(xml, "transformer-attention-mask").orElse(null); transformerOutput = getChildValue(xml, "transformer-output").orElse(null); - onnxExecutionMode = getChildValue(xml, "onnx-execution-mode").orElse(null); - onnxInteropThreads = getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt).orElse(null); - onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); - onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); model.registerOnnxModelCost(cluster); } @@ -79,10 +78,10 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo if (transformerStartSequenceToken != null) b.transformerStartSequenceToken(transformerStartSequenceToken); if (transformerEndSequenceToken != null) b.transformerEndSequenceToken(transformerEndSequenceToken); if (transformerMaskToken != null) b.transformerMaskToken(transformerMaskToken); - if (onnxExecutionMode != null) b.transformerExecutionMode( - ColBertEmbedderConfig.TransformerExecutionMode.Enum.valueOf(onnxExecutionMode)); - if (onnxInteropThreads != null) b.transformerInterOpThreads(onnxInteropThreads); - if (onnxIntraopThreads != null) b.transformerIntraOpThreads(onnxIntraopThreads); - if (onnxGpuDevice != null) b.transformerGpuDevice(onnxGpuDevice); + onnxModelOptions.executionMode().ifPresent(value -> b.transformerExecutionMode(TransformerExecutionMode.Enum.valueOf(value))); + onnxModelOptions.interOpThreads().ifPresent(b::transformerInterOpThreads); + onnxModelOptions.intraOpThreads().ifPresent(b::transformerIntraOpThreads); + onnxModelOptions.gpuDevice().ifPresent(value -> b.transformerGpuDevice(value.deviceNumber())); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java index 31b86142445..d1bd0dce000 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java @@ -1,5 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; @@ -8,6 +7,8 @@ import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.w3c.dom.Element; +import static com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig.PoolingStrategy; +import static com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig.TransformerExecutionMode; import static com.yahoo.text.XML.getChildValue; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; @@ -16,6 +17,8 @@ import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATI * @author bjorncs */ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEmbedderConfig.Producer { + + private final OnnxModelOptions onnxModelOptions; private final ModelReference modelRef; private final ModelReference vocabRef; private final Integer maxTokens; @@ -24,15 +27,16 @@ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEm private final String transformerTokenTypeIds; private final String transformerOutput; private final Boolean normalize; - private final String onnxExecutionMode; - private final Integer onnxInteropThreads; - private final Integer onnxIntraopThreads; - private final Integer onnxGpuDevice; private final String poolingStrategy; public HuggingFaceEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.huggingface.HuggingFaceEmbedder", INTEGRATION_BUNDLE_NAME, xml); var model = Model.fromXml(state, xml, "transformer-model").orElseThrow(); + this.onnxModelOptions = new OnnxModelOptions( + getChildValue(xml, "onnx-execution-mode"), + getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).map(OnnxModelOptions.GpuDevice::new)); modelRef = model.modelReference(); vocabRef = Model.fromXml(state, xml, "tokenizer-model") .map(Model::modelReference) @@ -43,10 +47,6 @@ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEm transformerTokenTypeIds = getChildValue(xml, "transformer-token-type-ids").orElse(null); transformerOutput = getChildValue(xml, "transformer-output").orElse(null); normalize = getChildValue(xml, "normalize").map(Boolean::parseBoolean).orElse(null); - onnxExecutionMode = getChildValue(xml, "onnx-execution-mode").orElse(null); - onnxInteropThreads = getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt).orElse(null); - onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); - onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); poolingStrategy = getChildValue(xml, "pooling-strategy").orElse(null); model.registerOnnxModelCost(cluster); } @@ -68,11 +68,11 @@ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEm if (transformerTokenTypeIds != null) b.transformerTokenTypeIds(transformerTokenTypeIds); if (transformerOutput != null) b.transformerOutput(transformerOutput); if (normalize != null) b.normalize(normalize); - if (onnxExecutionMode != null) b.transformerExecutionMode( - HuggingFaceEmbedderConfig.TransformerExecutionMode.Enum.valueOf(onnxExecutionMode)); - if (onnxInteropThreads != null) b.transformerInterOpThreads(onnxInteropThreads); - if (onnxIntraopThreads != null) b.transformerIntraOpThreads(onnxIntraopThreads); - if (onnxGpuDevice != null) b.transformerGpuDevice(onnxGpuDevice); - if (poolingStrategy != null) b.poolingStrategy(HuggingFaceEmbedderConfig.PoolingStrategy.Enum.valueOf(poolingStrategy)); + if (poolingStrategy != null) b.poolingStrategy(PoolingStrategy.Enum.valueOf(poolingStrategy)); + onnxModelOptions.executionMode().ifPresent(value -> b.transformerExecutionMode(TransformerExecutionMode.Enum.valueOf(value))); + onnxModelOptions.interOpThreads().ifPresent(b::transformerInterOpThreads); + onnxModelOptions.intraOpThreads().ifPresent(b::transformerIntraOpThreads); + onnxModelOptions.gpuDevice().ifPresent(value -> b.transformerGpuDevice(value.deviceNumber())); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/OnnxModelOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/OnnxModelOptions.java new file mode 100644 index 00000000000..6347f0dc427 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/OnnxModelOptions.java @@ -0,0 +1,45 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import java.util.Optional; + +/** + * Onnx model options that are relevant when deciding if an Onnx model needs to be reloaded. If any of the + * values in this class change, reload is needed. + * + * @author hmusum + */ +public record OnnxModelOptions(Optional<String> executionMode, Optional<Integer> interOpThreads, + Optional<Integer> intraOpThreads, Optional<GpuDevice> gpuDevice) { + + public static OnnxModelOptions empty() { + return new OnnxModelOptions(Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + } + + public OnnxModelOptions withExecutionMode(String executionMode) { + return new OnnxModelOptions(Optional.ofNullable(executionMode), interOpThreads, intraOpThreads, gpuDevice); + } + + public OnnxModelOptions withInterOpThreads(Integer interOpThreads) { + return new OnnxModelOptions(executionMode, Optional.ofNullable(interOpThreads), intraOpThreads, gpuDevice); + } + + public OnnxModelOptions withIntraOpThreads(Integer intraOpThreads) { + return new OnnxModelOptions(executionMode, interOpThreads, Optional.ofNullable(intraOpThreads), gpuDevice); + } + + public OnnxModelOptions withGpuDevice(GpuDevice gpuDevice) { + return new OnnxModelOptions(executionMode, interOpThreads, intraOpThreads, Optional.ofNullable(gpuDevice)); + } + + public record GpuDevice(int deviceNumber, boolean required) { + public GpuDevice { + if (deviceNumber < 0) throw new IllegalArgumentException("deviceNumber cannot be negative, got " + deviceNumber); + } + + public GpuDevice(int deviceNumber) { + this(deviceNumber, false); + } + } + +} diff --git a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg index 4600884b615..b62d7fd239c 100644 --- a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg +++ b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg @@ -17,6 +17,8 @@ rankprofile[].fef.property[].value "true" rankprofile[].fef.property[].name "vespa.type.attribute.aa" rankprofile[].fef.property[].value "tensor(d1[3])" rankprofile[].name "simple" +rankprofile[].fef.property[].name "vespa.type.feature.attribute(aa)" +rankprofile[].fef.property[].value "tensor(d1[3])" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg index cd8375cb68d..a72e9dc27cd 100644 --- a/config-model/src/test/derived/tensor/rank-profiles.cfg +++ b/config-model/src/test/derived/tensor/rank-profiles.cfg @@ -47,7 +47,7 @@ rankprofile[].name "profile2" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" -rankprofile[].fef.property[].value "reduce(reduce(join(attribute(f4), tensor(x[10],y[10],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x), sum)" +rankprofile[].fef.property[].value "reduce(reduce(join(attribute(f4), tensor(x[10],y[10],z[3])(((x == y) * (y == z))), f(a,b)(a * b)), sum, x), sum)" rankprofile[].fef.property[].name "vespa.type.attribute.f7" rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])" rankprofile[].fef.property[].name "vespa.type.attribute.f2" @@ -206,3 +206,22 @@ rankprofile[].fef.property[].name "vespa.type.query.para" rankprofile[].fef.property[].value "tensor<float>(p{})" rankprofile[].fef.property[].name "vespa.type.query.qvec" rankprofile[].fef.property[].value "tensor<float>(x[40])" +rankprofile[].name "with-just-unpack" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(firstphase)" +rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" +rankprofile[].fef.property[].value "reduce(query(para) * map_subspaces(attribute(f7), f(denseSubspaceInput)(tensor(x[40])((bit(denseSubspaceInput{x:(x / 8)},(x % 8)))))) * query(qvec), sum)" +rankprofile[].fef.property[].name "vespa.type.attribute.f7" +rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])" +rankprofile[].fef.property[].name "vespa.type.attribute.f2" +rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" +rankprofile[].fef.property[].name "vespa.type.attribute.f3" +rankprofile[].fef.property[].value "tensor(x{})" +rankprofile[].fef.property[].name "vespa.type.attribute.f4" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].name "vespa.type.attribute.f5" +rankprofile[].fef.property[].value "tensor<float>(x[10])" +rankprofile[].fef.property[].name "vespa.type.query.para" +rankprofile[].fef.property[].value "tensor(p{})" +rankprofile[].fef.property[].name "vespa.type.query.qvec" +rankprofile[].fef.property[].value "tensor(x[40])" diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd index 3a5fda3ac5d..a0f5cd92c56 100644 --- a/config-model/src/test/derived/tensor/tensor.sd +++ b/config-model/src/test/derived/tensor/tensor.sd @@ -135,4 +135,14 @@ schema tensor { } } + rank-profile with-just-unpack { + inputs { + query(para) tensor<double>(p{}) + query(qvec) tensor<double>(x[40]) + } + first-phase { + expression: sum(query(para)*unpack_bits(attribute(f7), double, little)*query(qvec)) + } + } + } diff --git a/configd/src/apps/sentinel/CMakeLists.txt b/configd/src/apps/sentinel/CMakeLists.txt index 607aba2785c..d3cfcff4135 100644 --- a/configd/src/apps/sentinel/CMakeLists.txt +++ b/configd/src/apps/sentinel/CMakeLists.txt @@ -14,6 +14,7 @@ vespa_add_executable(configd_config-sentinel_app output-connection.cpp outward-check.cpp peer-check.cpp + platform-specific.cpp report-connectivity.cpp rpchooks.cpp rpcserver.cpp diff --git a/configd/src/apps/sentinel/platform-specific.cpp b/configd/src/apps/sentinel/platform-specific.cpp new file mode 100644 index 00000000000..d1c3682e857 --- /dev/null +++ b/configd/src/apps/sentinel/platform-specific.cpp @@ -0,0 +1,45 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "platform-specific.h" +#include <vespa/vespalib/util/error.h> +#include <cstdlib> +#include <string_view> +#ifdef __linux__ +#include <sys/prctl.h> +#endif + +#include <vespa/log/log.h> +LOG_SETUP(".sentinel.platform-specific"); + +using namespace std::string_view_literals; + +namespace config::platform_specific { + +namespace { + +[[maybe_unused]] [[nodiscard]] +bool is_env_toggled(const char* var_name) { + const char* maybe_toggled = getenv(var_name); + return (maybe_toggled && (maybe_toggled == "true"sv || maybe_toggled == "yes"sv)); +} + +} + +void pledge_no_new_privileges_if_env_configured() { +#ifdef __linux__ + if (is_env_toggled("VESPA_PR_SET_NO_NEW_PRIVS")) { + // One-way toggle to prevent any subprocess from possibly getting extra privileges via + // setuid/setgid executables (modulo exciting things like kernel bugs or a small, trained + // rat that climbs into your computer and pulls an adorably tiny lever labeled "root access"). + // Helps mitigate a certain class of vulnerabilities, and also allows processes to install + // their own seccomp filters. + // See https://www.kernel.org/doc/Documentation/prctl/no_new_privs.txt + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) { + LOG(warning, "Failed to invoke prctl(PR_SET_NO_NEW_PRIVS): %s", vespalib::getErrorString(errno).c_str()); + } else { + LOG(debug, "Successfully invoked prctl(PR_SET_NO_NEW_PRIVS)"); + } + } +#endif +} + +} diff --git a/configd/src/apps/sentinel/platform-specific.h b/configd/src/apps/sentinel/platform-specific.h new file mode 100644 index 00000000000..d68d5f73768 --- /dev/null +++ b/configd/src/apps/sentinel/platform-specific.h @@ -0,0 +1,16 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace config::platform_specific { + +/** + * If running on Linux, sets the `no_new_privs` process bit, which amongst other + * things prevents all launched sub-process(es) from acquiring more privileges + * through setuid/setgid executables. + * + * Only takes effect if the `VESPA_PR_SET_NO_NEW_PRIVS` environment variable is + * set to "true" or "yes". + */ +void pledge_no_new_privileges_if_env_configured(); + +} diff --git a/configd/src/apps/sentinel/sentinel.cpp b/configd/src/apps/sentinel/sentinel.cpp index 59c690275c3..4f1d6019065 100644 --- a/configd/src/apps/sentinel/sentinel.cpp +++ b/configd/src/apps/sentinel/sentinel.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "manager.h" +#include "platform-specific.h" #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/util/signalhandler.h> #include <vespa/vespalib/util/exceptions.h> @@ -58,6 +59,8 @@ main(int argc, char **argv) } setlocale(LC_ALL, "C"); + platform_specific::pledge_no_new_privileges_if_env_configured(); // Affects all launched subprocesses + sentinel::Env environment; LOG(debug, "Reading configuration"); try { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java index 5a244b4b4b4..93c41f1b087 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.application; import com.google.common.io.ByteStreams; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.yolean.Exceptions; +import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; @@ -19,6 +20,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.zip.GZIPOutputStream; import static org.junit.Assert.assertEquals; @@ -33,13 +35,13 @@ public class CompressedApplicationInputStreamTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - private static void writeFileToTar(ArchiveOutputStream taos, File file) throws IOException { + private static <T extends ArchiveEntry> void writeFileToTar(ArchiveOutputStream<T> taos, File file) throws IOException { taos.putArchiveEntry(taos.createArchiveEntry(file, file.getName())); ByteStreams.copy(new FileInputStream(file), taos); taos.closeArchiveEntry(); } - private static File createArchiveFile(ArchiveOutputStream taos, File outFile) throws IOException { + private static <T extends ArchiveEntry> File createArchiveFile(ArchiveOutputStream<T> taos, File outFile) throws IOException { File app = new File("src/test/resources/deploy/validapp"); writeFileToTar(taos, new File(app, "services.xml")); writeFileToTar(taos, new File(app, "hosts.xml")); @@ -50,13 +52,13 @@ public class CompressedApplicationInputStreamTest { public static File createTarFile(Path dir) throws IOException { File outFile = Files.createTempFile(dir, "testapp", ".tar.gz").toFile(); - ArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(outFile))); + var archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(outFile))); return createArchiveFile(archiveOutputStream, outFile); } private File createZipFile(Path dir) throws IOException { File outFile = Files.createTempFile(dir, "testapp", ".tar.gz").toFile(); - ArchiveOutputStream archiveOutputStream = new ZipArchiveOutputStream(new FileOutputStream(outFile)); + var archiveOutputStream = new ZipArchiveOutputStream(new FileOutputStream(outFile)); return createArchiveFile(archiveOutputStream, outFile); } @@ -102,7 +104,7 @@ public class CompressedApplicationInputStreamTest { try (CompressedApplicationInputStream unpacked = streamFromTarGz(gzFile)) { outApp = unpacked.decompress(); } - List<File> files = Arrays.asList(outApp.listFiles()); + List<File> files = Arrays.asList(Objects.requireNonNull(outApp.listFiles())); assertEquals(5, files.size()); assertTrue(files.contains(new File(outApp, "services.xml"))); assertTrue(files.contains(new File(outApp, "hosts.xml"))); diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index fdc279f1e51..fc4ce0ff9f0 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -67,7 +67,7 @@ <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> <athenz.vespa.version>1.11.46</athenz.vespa.version> - <aws-sdk.vespa.version>1.12.580</aws-sdk.vespa.version> + <aws-sdk.vespa.version>1.12.590</aws-sdk.vespa.version> <!-- Athenz END --> <!-- WARNING: If you change curator version, you also need to update @@ -76,7 +76,7 @@ find zkfacade/src/main/java/org/apache/curator -name package-info.java | \ xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 5, minor = 3, micro = 0/g' --> - <bouncycastle.vespa.version>1.76</bouncycastle.vespa.version> + <bouncycastle.vespa.version>1.77</bouncycastle.vespa.version> <byte-buddy.vespa.version>1.14.9</byte-buddy.vespa.version> <checker-qual.vespa.version>3.38.0</checker-qual.vespa.version> <commons-beanutils.vespa.version>1.9.4</commons-beanutils.vespa.version> @@ -88,7 +88,7 @@ <commons-io.vespa.version>2.15.0</commons-io.vespa.version> <commons-lang3.vespa.version>3.13.0</commons-lang3.vespa.version> <commons.math3.vespa.version>3.6.1</commons.math3.vespa.version> - <commons-compress.vespa.version>1.24.0</commons-compress.vespa.version> + <commons-compress.vespa.version>1.25.0</commons-compress.vespa.version> <commons-cli.vespa.version>1.6.0</commons-cli.vespa.version> <curator.vespa.version>5.5.0</curator.vespa.version> <dropwizard.metrics.vespa.version>4.2.22</dropwizard.metrics.vespa.version> @@ -126,7 +126,7 @@ <org.json.vespa.version>20231013</org.json.vespa.version> <org.lz4.vespa.version>1.8.0</org.lz4.vespa.version> <prometheus.client.vespa.version>0.16.0</prometheus.client.vespa.version> - <protobuf.vespa.version>3.25.0</protobuf.vespa.version> + <protobuf.vespa.version>3.25.1</protobuf.vespa.version> <questdb.vespa.version>7.3.4</questdb.vespa.version> <spifly.vespa.version>1.3.7</spifly.vespa.version> <snappy.vespa.version>1.1.10.5</snappy.vespa.version> @@ -167,7 +167,7 @@ <maven-site-plugin.vespa.version>3.12.1</maven-site-plugin.vespa.version> <maven-source-plugin.vespa.version>3.3.0</maven-source-plugin.vespa.version> <properties-maven-plugin.vespa.version>1.2.1</properties-maven-plugin.vespa.version> - <versions-maven-plugin.vespa.version>2.16.1</versions-maven-plugin.vespa.version> + <versions-maven-plugin.vespa.version>2.16.2</versions-maven-plugin.vespa.version> </properties> <profiles> diff --git a/document/src/main/java/com/yahoo/document/annotation/Annotation.java b/document/src/main/java/com/yahoo/document/annotation/Annotation.java index 3d9300550ff..237ca6db58b 100644 --- a/document/src/main/java/com/yahoo/document/annotation/Annotation.java +++ b/document/src/main/java/com/yahoo/document/annotation/Annotation.java @@ -129,7 +129,7 @@ public class Annotation implements Comparable<Annotation> { } /** - * WARNING! Should only be used by deserializers! Sets the span node that this annotation points to. + * WARNING! Should only be used by deserializers! Sets the span node that this annotation points to. * * @param spanNode the span node that this annotation shall point to. */ @@ -221,10 +221,9 @@ public class Annotation implements Comparable<Annotation> { @Override public String toString() { - String retval = "annotation of type " + type; - retval += ((value == null) ? " (no value)" : " (with value)"); - retval += ((spanNode == null) ? " (no span)" : (" with span "+spanNode)); - return retval; + return type + " annotation " + + ((value == null) ? " (no value)" : " (with value)") + + ((spanNode == null) ? " (no span)" : (" with span "+spanNode)); } diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java index 6e2b986a478..ac2e6aefa1b 100644 --- a/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java +++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java @@ -23,11 +23,7 @@ abstract class AnnotationContainer { */ abstract void annotate(Annotation annotation); - /** - * Returns a mutable collection of annotations. - * - * @return a mutable collection of annotations. - */ + /** Returns a mutable collection of the annotations in this. */ abstract Collection<Annotation> annotations(); /** diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java index d8709baa3a1..01b8f990bb4 100644 --- a/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java +++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java @@ -16,6 +16,7 @@ import java.util.NoSuchElementException; */ // TODO: Should this be removed? public class AnnotationType2AnnotationContainer extends IteratingAnnotationContainer { + private final Multimap<AnnotationType, Annotation> annotationType2Annotation = Multimaps.newMultimap(new IdentityHashMap<>(), ArrayList::new); @Override @@ -31,7 +32,6 @@ public class AnnotationType2AnnotationContainer extends IteratingAnnotationConta } @Override - @SuppressWarnings("unchecked") Collection<Annotation> annotations() { return annotationType2Annotation.values(); } @@ -56,12 +56,12 @@ public class AnnotationType2AnnotationContainer extends IteratingAnnotationConta } private class NonRecursiveIterator implements Iterator<Annotation> { + private final IdentityHashMap<SpanNode, SpanNode> nodes; private final Iterator<Annotation> annotationIt; private Annotation next = null; private boolean nextCalled; - @SuppressWarnings("unchecked") public NonRecursiveIterator(IdentityHashMap<SpanNode, SpanNode> nodes) { this.nodes = nodes; this.annotationIt = annotationType2Annotation.values().iterator(); @@ -106,4 +106,5 @@ public class AnnotationType2AnnotationContainer extends IteratingAnnotationConta nextCalled = false; } } + } diff --git a/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java index 54af80e6e43..c7c3545fc09 100644 --- a/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java +++ b/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java @@ -5,30 +5,29 @@ import java.util.IdentityHashMap; import java.util.Iterator; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ abstract class IteratingAnnotationContainer extends AnnotationContainer { @Override Iterator<Annotation> iterator(SpanNode node) { - IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<SpanNode, SpanNode>(); + IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<>(); nodes.put(node, node); return iterator(nodes); } @Override Iterator<Annotation> iteratorRecursive(SpanNode node) { - IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<SpanNode, SpanNode>(); + IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<>(); nodes.put(node, node); - { - Iterator<SpanNode> childrenIt = node.childIteratorRecursive(); - while (childrenIt.hasNext()) { - SpanNode child = childrenIt.next(); - nodes.put(child, child); - } + Iterator<SpanNode> childrenIt = node.childIteratorRecursive(); + while (childrenIt.hasNext()) { + SpanNode child = childrenIt.next(); + nodes.put(child, child); } return iterator(nodes); } abstract Iterator<Annotation> iterator(IdentityHashMap<SpanNode, SpanNode> nodes); + } diff --git a/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java index c2c22558a32..b8d8cd692d8 100644 --- a/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java +++ b/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java @@ -13,6 +13,7 @@ import java.util.NoSuchElementException; * @author Einar M R Rosenvinge */ public class ListAnnotationContainer extends IteratingAnnotationContainer { + private final List<Annotation> annotations = new LinkedList<>(); @Override @@ -38,9 +39,8 @@ public class ListAnnotationContainer extends IteratingAnnotationContainer { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof ListAnnotationContainer)) return false; - ListAnnotationContainer that = (ListAnnotationContainer) o; - if (!annotations.equals(that.annotations)) return false; + if (!(o instanceof ListAnnotationContainer other)) return false; + if (!annotations.equals(other.annotations)) return false; return true; } @@ -50,8 +50,9 @@ public class ListAnnotationContainer extends IteratingAnnotationContainer { } private class AnnotationIterator implements Iterator<Annotation> { - private IdentityHashMap<SpanNode, SpanNode> nodes; - private PeekableListIterator<Annotation> base; + + private final IdentityHashMap<SpanNode, SpanNode> nodes; + private final PeekableListIterator<Annotation> base; private boolean nextCalled = false; AnnotationIterator(ListIterator<Annotation> baseIt, IdentityHashMap<SpanNode, SpanNode> nodes) { @@ -91,4 +92,5 @@ public class ListAnnotationContainer extends IteratingAnnotationContainer { nextCalled = false; } } + } diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanList.java b/document/src/main/java/com/yahoo/document/annotation/SpanList.java index 9530689c3f1..04bdf5c892e 100644 --- a/document/src/main/java/com/yahoo/document/annotation/SpanList.java +++ b/document/src/main/java/com/yahoo/document/annotation/SpanList.java @@ -11,13 +11,14 @@ import java.util.ListIterator; /** * A node in a Span tree that can have child nodes. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class SpanList extends SpanNode { + public static final byte ID = 2; private final List<SpanNode> children; - private int cachedFrom = Integer.MIN_VALUE; //triggers calculateFrom() - private int cachedTo = Integer.MIN_VALUE; //triggers calculateTo() + private int cachedFrom = Integer.MIN_VALUE; // triggers calculateFrom() + private int cachedTo = Integer.MIN_VALUE; // triggers calculateTo() /** Creates a new SpanList. */ public SpanList() { @@ -39,7 +40,7 @@ public class SpanList extends SpanNode { * @param other the SpanList to copy. */ public SpanList(SpanList other) { - this.children = new LinkedList<SpanNode>(); + this.children = new LinkedList<>(); for (SpanNode otherNode : other.children) { if (otherNode instanceof Span) { children.add(new Span((Span) otherNode)); @@ -86,15 +87,15 @@ public class SpanList extends SpanNode { /** Create a span, add it to this list and return it */ public Span span(int from, int length) { - Span span=new Span(from,length); + Span span = new Span(from, length); add(span); return span; } void setInvalid() { - //invalidate ourselves: + // invalidate ourselves: super.setInvalid(); - //invalidate all our children: + // invalidate all our children: for (SpanNode node : children()) { node.setInvalid(); } @@ -213,20 +214,12 @@ public class SpanList extends SpanNode { return this; } - /** - * Returns a <strong>modifiable</strong> list of the immediate children of this SpanList. - * - * @return a <strong>modifiable</strong> list of the immediate children of this SpanList. - */ + /** Returns a modifiable list of the immediate children of this SpanList. */ protected List<SpanNode> children() { return children; } - /** - * Returns the number of children this SpanList holds. - * - * @return the number of children this SpanList holds. - */ + /** Returns the number of children this SpanList holds. */ public int numChildren() { return children().size(); } @@ -394,11 +387,9 @@ public class SpanList extends SpanNode { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof SpanList)) return false; + if (!(o instanceof SpanList spanList)) return false; if (!super.equals(o)) return false; - SpanList spanList = (SpanList) o; - if (children() != null ? !children().equals(spanList.children()) : spanList.children() != null) return false; return true; @@ -415,4 +406,5 @@ public class SpanList extends SpanNode { public String toString() { return "SpanList with " + children().size() + " children"; } + } diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java index a66639b3bfd..f563fd0d6cd 100644 --- a/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java +++ b/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java @@ -18,6 +18,7 @@ import java.util.List; * @author Einar M R Rosenvinge */ class SpanNode2AnnotationContainer extends AnnotationContainer { + private final Multimap<SpanNode, Annotation> spanNode2Annotation = Multimaps.newMultimap(new IdentityHashMap<>(), ArrayList::new); @Override diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanTree.java b/document/src/main/java/com/yahoo/document/annotation/SpanTree.java index f785cf3b3ec..2faf17ce428 100644 --- a/document/src/main/java/com/yahoo/document/annotation/SpanTree.java +++ b/document/src/main/java/com/yahoo/document/annotation/SpanTree.java @@ -19,6 +19,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * A SpanTree holds a root node of a tree of SpanNodes, and a List of Annotations pointing to these nodes @@ -431,7 +432,7 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl } /** - * Adds an Annotation to the internal list of annotations for this SpanTree. Use this when + * Adds an Annotation to the internal list of annotations for this SpanTree. Use this when * adding an Annotation that shall annotate a SpanNode. Upon return, Annotation.getSpanNode() * returns the given node. * @@ -446,7 +447,7 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl } /** - * Adds an Annotation to the internal list of annotations for this SpanTree. Use this when + * Adds an Annotation to the internal list of annotations for this SpanTree. Use this when * adding an Annotation that shall annotate a SpanNode. Upon return, Annotation.getSpanNode() * returns the given node. This one is unchecked and assumes that the SpanNode is valid and has * already been attached to the Annotation. @@ -538,10 +539,7 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl } } - /** - * Returns an Iterator over all annotations in this tree. Note that the iteration order is non-deterministic. - * @return an Iterator over all annotations in this tree. - */ + /** Returns an Iterator over all annotations in this tree. Note that the iteration order is non-deterministic. */ public Iterator<Annotation> iterator() { return annotations.annotations().iterator(); } @@ -641,7 +639,9 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl @Override public String toString() { - return "SpanTree '" + name + "'"; + return "SpanTree '" + name + "' with root: " + root + + ( annotations.annotations().size() > 5 ? "" : + ", annotations: " + annotations.annotations().stream().map(Annotation::toString).collect(Collectors.joining(", "))); } @Override diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java index 0b1bbf5d3ca..bb54b41069b 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Struct.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java @@ -218,8 +218,7 @@ public class Struct extends StructuredFieldValue { StringBuilder retVal = new StringBuilder(); retVal.append("Struct (").append(getDataType()).append("): "); int [] increasing = getInOrder(); - for (int i = 0; i < increasing.length; i++) { - int id = increasing[i]; + for (int id : increasing) { retVal.append(getDataType().getField(id)).append("=").append(values.get(id)).append(", "); } return retVal.toString(); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java index bcfeddacff3..080db640dfe 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java @@ -6,7 +6,6 @@ import com.yahoo.compress.ZstdOutputStream; import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.ArchiveInputStream; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; @@ -18,7 +17,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.List; import java.util.Objects; import java.util.logging.Level; @@ -52,24 +50,23 @@ public class FileReferenceCompressor { } public File compress(File directory, File outputFile) throws IOException { - return compress(directory, - Files.find(Paths.get(directory.getAbsolutePath()), - recurseDepth, - (p, basicFileAttributes) -> basicFileAttributes.isRegularFile()) - .map(Path::toFile).toList(), - outputFile); + try (var paths = Files.find(Path.of(directory.getAbsolutePath()), recurseDepth, + (p, basicFileAttributes) -> basicFileAttributes.isRegularFile())) + { + return compress(directory, paths.map(Path::toFile).toList(), outputFile); + } } public void decompress(File inputFile, File outputDir) throws IOException { log.log(Level.FINEST, () -> "Decompressing '" + inputFile + "' into '" + outputDir + "'"); - try (ArchiveInputStream ais = new TarArchiveInputStream(decompressedInputStream(inputFile))) { + try (TarArchiveInputStream ais = new TarArchiveInputStream(decompressedInputStream(inputFile))) { decompress(ais, outputDir); } catch (IllegalArgumentException e) { throw new RuntimeException("Unable to decompress '" + inputFile.getAbsolutePath() + "': " + e.getMessage()); } } - private static void decompress(ArchiveInputStream archiveInputStream, File outputFile) throws IOException { + private static void decompress(TarArchiveInputStream archiveInputStream, File outputFile) throws IOException { int entries = 0; ArchiveEntry entry; while ((entry = archiveInputStream.getNextEntry()) != null) { @@ -99,7 +96,7 @@ public class FileReferenceCompressor { } } - private static void createArchiveFile(ArchiveOutputStream archiveOutputStream, File baseDir, List<File> inputFiles) throws IOException { + private static <T extends ArchiveEntry> void createArchiveFile(ArchiveOutputStream<T> archiveOutputStream, File baseDir, List<File> inputFiles) throws IOException { inputFiles.forEach(file -> { try { writeFileToTar(archiveOutputStream, baseDir, file); @@ -110,7 +107,7 @@ public class FileReferenceCompressor { archiveOutputStream.close(); } - private static void writeFileToTar(ArchiveOutputStream taos, File baseDir, File file) throws IOException { + private static <T extends ArchiveEntry> void writeFileToTar(ArchiveOutputStream<T> taos, File baseDir, File file) throws IOException { taos.putArchiveEntry(taos.createArchiveEntry(file, baseDir.toPath().relativize(file.toPath()).toString())); try (FileInputStream inputStream = new FileInputStream(file)) { inputStream.transferTo(taos); @@ -119,35 +116,33 @@ public class FileReferenceCompressor { } private OutputStream compressedOutputStream(File outputFile) throws IOException { - switch (type) { - case compressed: + return switch (type) { + case compressed -> { log.log(Level.FINEST, () -> "Compressing with compression type " + compressionType); - return switch (compressionType) { + yield switch (compressionType) { case gzip -> new GZIPOutputStream(new FileOutputStream(outputFile)); case lz4 -> new LZ4BlockOutputStream(new FileOutputStream(outputFile)); case zstd -> new ZstdOutputStream(new FileOutputStream(outputFile)); }; - case file: - return new FileOutputStream(outputFile); - default: - throw new RuntimeException("Unknown file reference type " + type); - } + } + case file -> new FileOutputStream(outputFile); + default -> throw new RuntimeException("Unknown file reference type " + type); + }; } private InputStream decompressedInputStream(File inputFile) throws IOException { - switch (type) { - case compressed: + return switch (type) { + case compressed -> { log.log(Level.FINEST, () -> "Decompressing with compression type " + compressionType); - return switch (compressionType) { + yield switch (compressionType) { case gzip -> new GZIPInputStream(new FileInputStream(inputFile)); case lz4 -> new LZ4BlockInputStream(new FileInputStream(inputFile)); case zstd -> new ZstdInputStream(new FileInputStream(inputFile)); }; - case file: - return new FileInputStream(inputFile); - default: - throw new RuntimeException("Unknown file reference type " + type); - } + } + case file -> new FileInputStream(inputFile); + default -> throw new RuntimeException("Unknown file reference type " + type); + }; } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 704155b3569..7a7087fb434 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -302,11 +302,6 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); - public static final UnboundBooleanFlag ENABLE_CROWDSTRIKE = defineFeatureFlag( - "enable-crowdstrike", true, List.of("andreer"), "2023-04-13", "2023-11-14", - "Whether to enable CrowdStrike.", "Takes effect on next host admin tick", - HOSTNAME); - public static final UnboundBooleanFlag ENABLE_THE_ONE_THAT_SHOULD_NOT_BE_NAMED = defineFeatureFlag( "enable-the-one-that-should-not-be-named", false, List.of("hmusum"), "2023-05-08", "2023-12-01", "Whether to enable the one program that should not be named", @@ -325,18 +320,6 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); - public static final UnboundBooleanFlag EXCLUSIVE_PROVISIONING = defineFeatureFlag( - "exclusive-provisioning", true, - List.of("hakonhall"), "2023-10-12", "2023-12-20", - "Unused, remove once Vespa >=8.257 has rolled out everywhere", - "no-op"); - - public static final UnboundBooleanFlag MAKE_EXCLUSIVE = defineFeatureFlag( - "make-exclusive", true, - List.of("hakonhall"), "2023-10-20", "2023-12-20", - "Unused, remove once Vespa >=8.257 has rolled out everywhere", - "no-op"); - public static final UnboundBooleanFlag WRITE_CONFIG_SERVER_SESSION_DATA_AS_ONE_BLOB = defineFeatureFlag( "write-config-server-session-data-as-blob", false, List.of("hmusum"), "2023-07-19", "2024-01-01", @@ -351,21 +334,21 @@ public class Flags { public static final UnboundBooleanFlag MORE_WIREGUARD = defineFeatureFlag( "more-wireguard", false, - List.of("andreer"), "2023-08-21", "2023-11-14", + List.of("andreer"), "2023-08-21", "2023-12-14", "Use wireguard in INternal enCLAVES", "Takes effect on next host-admin run", HOSTNAME, CLOUD_ACCOUNT); public static final UnboundBooleanFlag IPV6_AWS_TARGET_GROUPS = defineFeatureFlag( "ipv6-aws-target-groups", false, - List.of("andreer"), "2023-08-28", "2023-11-14", + List.of("andreer"), "2023-08-28", "2023-12-14", "Always use IPv6 target groups for load balancers in aws", "Takes effect on next load-balancer provisioning", HOSTNAME, CLOUD_ACCOUNT); public static final UnboundBooleanFlag PROVISION_IPV6_ONLY_AWS = defineFeatureFlag( "provision-ipv6-only", false, - List.of("andreer"), "2023-08-28", "2023-11-14", + List.of("andreer"), "2023-08-28", "2023-12-14", "Provision without private IPv4 addresses in INternal enCLAVES in AWS", "Takes effect on next host provisioning / run of host-admin", HOSTNAME, CLOUD_ACCOUNT); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java index 26058eeb8f3..fdfadf65400 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java @@ -15,6 +15,8 @@ import com.yahoo.vespa.indexinglanguage.linguistics.LinguisticsAnnotator; import java.util.Iterator; +import static com.yahoo.language.LinguisticsCase.toLowerCase; + /** * A filter which splits incoming text into n-grams. * @@ -68,8 +70,9 @@ public final class NGramExpression extends Expression { // annotate gram as a word term String gramString = gram.extractFrom(output.getString()); - typedSpan(gram.getStart(), gram.getCodePointCount(), TokenType.ALPHABETIC, spanList). - annotate(LinguisticsAnnotator.lowerCaseTermAnnotation(gramString, gramString)); + typedSpan(gram.getStart(), + gram.getCodePointCount(), + TokenType.ALPHABETIC, spanList).annotate(LinguisticsAnnotator.termAnnotation(toLowerCase(gramString), gramString)); lastPosition = gram.getStart() + gram.getCodePointCount(); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java index 684bae3bf97..5c1bf0813c4 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java @@ -13,7 +13,7 @@ public class AnnotatorConfig implements Cloneable { private Language language; private StemMode stemMode; private boolean removeAccents; - private int maxTermOccurences; + private int maxTermOccurrences; private int maxTokenizeLength; public static final int DEFAULT_MAX_TERM_OCCURRENCES; @@ -29,7 +29,7 @@ public class AnnotatorConfig implements Cloneable { language = Language.ENGLISH; stemMode = StemMode.NONE; removeAccents = false; - maxTermOccurences = DEFAULT_MAX_TERM_OCCURRENCES; + maxTermOccurrences = DEFAULT_MAX_TERM_OCCURRENCES; maxTokenizeLength = DEFAULT_MAX_TOKENIZE_LENGTH; } @@ -37,7 +37,7 @@ public class AnnotatorConfig implements Cloneable { language = rhs.language; stemMode = rhs.stemMode; removeAccents = rhs.removeAccents; - maxTermOccurences = rhs.maxTermOccurences; + maxTermOccurrences = rhs.maxTermOccurrences; maxTokenizeLength = rhs.maxTokenizeLength; } @@ -74,11 +74,11 @@ public class AnnotatorConfig implements Cloneable { } public int getMaxTermOccurrences() { - return maxTermOccurences; + return maxTermOccurrences; } public AnnotatorConfig setMaxTermOccurrences(int maxTermCount) { - this.maxTermOccurences = maxTermCount; + this.maxTermOccurrences = maxTermCount; return this; } @@ -109,7 +109,7 @@ public class AnnotatorConfig implements Cloneable { if (removeAccents != rhs.removeAccents) { return false; } - if (maxTermOccurences != rhs.maxTermOccurences) { + if (maxTermOccurrences != rhs.maxTermOccurrences) { return false; } if (maxTokenizeLength != rhs.maxTokenizeLength) { @@ -121,6 +121,7 @@ public class AnnotatorConfig implements Cloneable { @Override public int hashCode() { return getClass().hashCode() + language.hashCode() + stemMode.hashCode() + - Boolean.valueOf(removeAccents).hashCode() + maxTermOccurences + maxTokenizeLength; + Boolean.valueOf(removeAccents).hashCode() + maxTermOccurrences + maxTokenizeLength; } + } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java index 191d067effe..52cd8a8ff54 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java @@ -34,8 +34,8 @@ public class LinguisticsAnnotator { final Map<String, Integer> termOccurrences = new HashMap<>(); final int maxOccurrences; - public TermOccurrences(int maxOccurences) { - this.maxOccurrences = maxOccurences; + public TermOccurrences(int maxOccurrences) { + this.maxOccurrences = maxOccurrences; } boolean termCountBelowLimit(String term) { @@ -86,24 +86,23 @@ public class LinguisticsAnnotator { } /** - * Creates a TERM annotation which has the lowercase value as annotation (only) if it is different from the + * Creates a TERM annotation which has the term as annotation (only) if it is different from the * original. * - * @param termToLowerCase the term to lower case - * @param origTerm the original term + * @param term the term + * @param origTerm the original term * @return the created TERM annotation */ - public static Annotation lowerCaseTermAnnotation(String termToLowerCase, String origTerm) { - String annotationValue = toLowerCase(termToLowerCase); - if (annotationValue.equals(origTerm)) { + public static Annotation termAnnotation(String term, String origTerm) { + if (term.equals(origTerm)) return new Annotation(AnnotationTypes.TERM); - } - return new Annotation(AnnotationTypes.TERM, new StringFieldValue(annotationValue)); + else + return new Annotation(AnnotationTypes.TERM, new StringFieldValue(term)); } private static void addAnnotation(Span here, String term, String orig, TermOccurrences termOccurrences) { if (termOccurrences.termCountBelowLimit(term)) { - here.annotate(lowerCaseTermAnnotation(term, orig)); + here.annotate(termAnnotation(term, orig)); } } @@ -127,29 +126,24 @@ public class LinguisticsAnnotator { } if (mode == StemMode.ALL) { Span where = parent.span((int)token.getOffset(), token.getOrig().length()); - String lowercasedOrig = toLowerCase(token.getOrig()); - addAnnotation(where, token.getOrig(), token.getOrig(), termOccurrences); - String lowercasedTerm = lowercasedOrig; + String lowercasedOrig = toLowerCase(token.getOrig()); String term = token.getTokenString(); if (term != null) { - lowercasedTerm = toLowerCase(term); - } - if (! lowercasedOrig.equals(lowercasedTerm)) { addAnnotation(where, term, token.getOrig(), termOccurrences); + if ( ! term.equals(lowercasedOrig)) + addAnnotation(where, token.getOrig(), token.getOrig(), termOccurrences); } for (int i = 0; i < token.getNumStems(); i++) { String stem = token.getStem(i); - String lowercasedStem = toLowerCase(stem); - if (! (lowercasedOrig.equals(lowercasedStem) || lowercasedTerm.equals(lowercasedStem))) { + if (! (stem.equals(lowercasedOrig) || stem.equals(term))) addAnnotation(where, stem, token.getOrig(), termOccurrences); - } } } else { String term = token.getTokenString(); if (term == null || term.trim().isEmpty()) return; if (termOccurrences.termCountBelowLimit(term)) { - parent.span((int)token.getOffset(), token.getOrig().length()).annotate(lowerCaseTermAnnotation(term, token.getOrig())); + parent.span((int)token.getOffset(), token.getOrig().length()).annotate(termAnnotation(term, token.getOrig())); } } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java index bcde8751de8..b4e266ab3eb 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java @@ -57,8 +57,8 @@ public class NGramTestCase { new NGramExpression(new SimpleLinguistics(), 3).execute(context); StringFieldValue value = (StringFieldValue)context.getValue(); - assertEquals("Grams are pure annotations - field value is unchanged", "en gul Bille sang... ", - value.getString()); + assertEquals("Grams are pure annotations - field value is unchanged", + "en gul Bille sang... ", value.getString()); SpanTree gramTree = value.getSpanTree(SpanTrees.LINGUISTICS); assertNotNull(gramTree); SpanList grams = (SpanList)gramTree.getRoot(); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java index 67bff3843ee..a4dbe1fe826 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java @@ -19,7 +19,6 @@ import org.junit.Test; import org.mockito.Mockito; import java.util.*; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -29,8 +28,6 @@ import static org.junit.Assert.assertTrue; */ public class LinguisticsAnnotatorTestCase { - private static final AnnotatorConfig CONFIG = new AnnotatorConfig(); - @Test public void requireThatAnnotateFailsWithZeroTokens() { assertAnnotations(null, "foo"); @@ -42,7 +39,7 @@ public class LinguisticsAnnotatorTestCase { if (type.isIndexable()) { continue; } - assertAnnotations(null, "foo", newToken("foo", "bar", type)); + assertAnnotations(null, "foo", token("foo", "bar", type)); } } @@ -54,7 +51,27 @@ public class LinguisticsAnnotatorTestCase { if (!type.isIndexable()) { continue; } - assertAnnotations(expected, "foo", newToken("foo", "bar", type)); + assertAnnotations(expected, "foo", token("foo", "bar", type)); + } + } + + @Test + public void requireThatIndexableTokenStringsAreAnnotatedWithModeALL() { + SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); + expected.spanList().span(0, 5).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("tesla"))); + var span2 = expected.spanList().span(0, 4); + span2.annotate(new Annotation(AnnotationTypes.TERM)); + span2.annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("car"))); + var span3 = expected.spanList().span(0, 8); + span3.annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("modelxes"))); + span3.annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("modelx"))); + span3.annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("mex"))); + for (TokenType type : TokenType.values()) { + if (!type.isIndexable()) continue; + assertAnnotations(expected, "Tesla cars", new AnnotatorConfig().setStemMode("ALL"), + token("Tesla", "tesla", type), + token("cars", "car", type), + SimpleToken.fromStems("ModelXes", List.of("modelxes", "modelx", "mex"))); } } @@ -63,7 +80,7 @@ public class LinguisticsAnnotatorTestCase { SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); for (TokenType type : TokenType.values()) { - assertAnnotations(expected, "foo", newToken("foo", "bar", type, true)); + assertAnnotations(expected, "foo", token("foo", "bar", type, true)); } } @@ -76,21 +93,21 @@ public class LinguisticsAnnotatorTestCase { if (!specialToken && !type.isIndexable()) { continue; } - assertAnnotations(expected, "foo", newToken("foo", "foo", type, specialToken)); + assertAnnotations(expected, "foo", token("foo", "foo", type, specialToken)); } } } @Test - public void requireThatTermAnnotationsAreLowerCased() { + public void requireThatTermAnnotationsPreserveCasing() { SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); - expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); + expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("BaR"))); for (boolean specialToken : Arrays.asList(true, false)) { for (TokenType type : TokenType.values()) { if (!specialToken && !type.isIndexable()) { continue; } - assertAnnotations(expected, "foo", newToken("foo", "BAR", type, specialToken)); + assertAnnotations(expected, "foo", token("foo", "BaR", type, specialToken)); } } } @@ -102,11 +119,11 @@ public class LinguisticsAnnotatorTestCase { expected.spanList().span(3, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); expected.spanList().span(6, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("baz"))); - SimpleToken token = newToken("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC) - .addComponent(newToken("FOO", "foo", TokenType.ALPHABETIC).setOffset(0)) - .addComponent(newToken("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3) - .addComponent(newToken("BAR", "bar", TokenType.ALPHABETIC).setOffset(3)) - .addComponent(newToken("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6))); + SimpleToken token = token("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC) + .addComponent(token("FOO", "foo", TokenType.ALPHABETIC).setOffset(0)) + .addComponent(token("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3) + .addComponent(token("BAR", "bar", TokenType.ALPHABETIC).setOffset(3)) + .addComponent(token("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6))); assertAnnotations(expected, "foobarbaz", token); } @@ -116,11 +133,11 @@ public class LinguisticsAnnotatorTestCase { expected.spanList().span(0, 9).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("foobarbaz"))); - SimpleToken token = newToken("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC).setSpecialToken(true) - .addComponent(newToken("FOO", "foo", TokenType.ALPHABETIC).setOffset(0)) - .addComponent(newToken("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3) - .addComponent(newToken("BAR", "bar", TokenType.ALPHABETIC).setOffset(3)) - .addComponent(newToken("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6))); + SimpleToken token = token("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC).setSpecialToken(true) + .addComponent(token("FOO", "foo", TokenType.ALPHABETIC).setOffset(0)) + .addComponent(token("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3) + .addComponent(token("BAR", "bar", TokenType.ALPHABETIC).setOffset(3)) + .addComponent(token("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6))); assertAnnotations(expected, "foobarbaz", token); } @@ -140,7 +157,8 @@ public class LinguisticsAnnotatorTestCase { continue; } assertAnnotations(expected, "foo", - newLinguistics(List.of(newToken("foo", "foo", type, specialToken)), + new AnnotatorConfig(), + newLinguistics(List.of(token("foo", "foo", type, specialToken)), Collections.singletonMap("foo", "bar"))); } } @@ -154,11 +172,9 @@ public class LinguisticsAnnotatorTestCase { StringFieldValue val = new StringFieldValue("foo"); val.setSpanTree(spanTree); - Linguistics linguistics = newLinguistics(List.of(newToken("foo", "bar", TokenType.ALPHABETIC, false)), + Linguistics linguistics = newLinguistics(List.of(token("foo", "bar", TokenType.ALPHABETIC, false)), Collections.<String, String>emptyMap()); - new LinguisticsAnnotator(linguistics, CONFIG).annotate(val); - - assertTrue(new LinguisticsAnnotator(linguistics, CONFIG).annotate(val)); + assertTrue(new LinguisticsAnnotator(linguistics, new AnnotatorConfig()).annotate(val)); assertEquals(spanTree, val.getSpanTree(SpanTrees.LINGUISTICS)); } @@ -186,7 +202,7 @@ public class LinguisticsAnnotatorTestCase { } @Test - public void requireThatMaxTermOccurencesIsHonored() { + public void requireThatMaxTermOccurrencesIsHonored() { final String inputTerm = "foo"; final String stemmedInputTerm = "bar"; // completely different from // inputTerm for safer test @@ -204,7 +220,7 @@ public class LinguisticsAnnotatorTestCase { StringBuilder input = new StringBuilder(); Token[] tokens = new Token[inputTermOccurence]; for (int i = 0; i < inputTermOccurence; ++i) { - SimpleToken t = newToken(inputTerm, stemmedInputTerm, type); + SimpleToken t = token(inputTerm, stemmedInputTerm, type); t.setOffset(i * paddedInputTerm.length()); tokens[i] = t; input.append(paddedInputTerm); @@ -214,28 +230,29 @@ public class LinguisticsAnnotatorTestCase { } // -------------------------------------------------------------------------------- - // // Utilities - // - // -------------------------------------------------------------------------------- - private static SimpleToken newToken(String orig, String stem, TokenType type) { - return newToken(orig, stem, type, false); + private static SimpleToken token(String orig, String stem, TokenType type) { + return token(orig, stem, type, false); } - private static SimpleToken newToken(String orig, String stem, TokenType type, boolean specialToken) { + private static SimpleToken token(String orig, String stem, TokenType type, boolean specialToken) { return new SimpleToken(orig).setTokenString(stem) .setType(type) .setSpecialToken(specialToken); } private static void assertAnnotations(SpanTree expected, String value, Token... tokens) { - assertAnnotations(expected, value, newLinguistics(Arrays.asList(tokens), Collections.<String, String>emptyMap())); + assertAnnotations(expected, value, new AnnotatorConfig(), newLinguistics(Arrays.asList(tokens), Collections.emptyMap())); + } + + private static void assertAnnotations(SpanTree expected, String value, AnnotatorConfig config, Token... tokens) { + assertAnnotations(expected, value, config, newLinguistics(Arrays.asList(tokens), Collections.emptyMap())); } - private static void assertAnnotations(SpanTree expected, String str, Linguistics linguistics) { + private static void assertAnnotations(SpanTree expected, String str, AnnotatorConfig config, Linguistics linguistics) { StringFieldValue val = new StringFieldValue(str); - assertEquals(expected != null, new LinguisticsAnnotator(linguistics, CONFIG).annotate(val)); + assertEquals(expected != null, new LinguisticsAnnotator(linguistics, config).annotate(val)); assertEquals(expected, val.getSpanTree(SpanTrees.LINGUISTICS)); } diff --git a/jdisc-cloud-aws/pom.xml b/jdisc-cloud-aws/pom.xml index 3dfbf1e9154..af5c5c7cc8f 100644 --- a/jdisc-cloud-aws/pom.xml +++ b/jdisc-cloud-aws/pom.xml @@ -47,6 +47,23 @@ <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-ssm</artifactId> </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>testutil</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + </dependencies> <build> diff --git a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java index fc9c03a824a..6de8b5c0142 100644 --- a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java +++ b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java @@ -14,45 +14,87 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; +import java.util.logging.Logger; public class VespaAwsCredentialsProvider implements AWSCredentialsProvider { + private static final Logger logger = Logger.getLogger(VespaAwsCredentialsProvider.class.getName()); private static final String DEFAULT_CREDENTIALS_PATH = "/opt/vespa/var/vespa/aws/credentials.json"; - - private final AtomicReference<AWSCredentials> credentials = new AtomicReference<>(); + private static final Duration REFRESH_INTERVAL = Duration.ofMinutes(30); + private final AtomicReference<Credentials> credentials = new AtomicReference<>(); private final Path credentialsPath; - + private final Clock clock; public VespaAwsCredentialsProvider() { - this.credentialsPath = Path.of(DEFAULT_CREDENTIALS_PATH); + this(Path.of(DEFAULT_CREDENTIALS_PATH), Clock.systemUTC()); + } + + VespaAwsCredentialsProvider(Path credentialsPath, Clock clock) { + this.credentialsPath = credentialsPath; + this.clock = clock; refresh(); } @Override public AWSCredentials getCredentials() { - return credentials.get(); + Credentials sessionCredentials = credentials.get(); + if (Duration.between(clock.instant(), sessionCredentials.expiry).abs().compareTo(REFRESH_INTERVAL)<0) { + refresh(); + sessionCredentials = credentials.get(); + } + return sessionCredentials; } @Override public void refresh() { try { + logger.log(Level.FINE, "Refreshing credentials from disk"); credentials.set(readCredentials()); } catch (Exception e) { throw new RuntimeException("Unable to get credentials. Please ensure cluster is configured as exclusive. See: https://cloud.vespa.ai/en/reference/services#nodes"); } } - private AWSSessionCredentials readCredentials() { + private Credentials readCredentials() { try { Slime slime = SlimeUtils.jsonToSlime(Files.readAllBytes(credentialsPath)); Cursor cursor = slime.get(); String accessKey = cursor.field("awsAccessKey").asString(); String secretKey = cursor.field("awsSecretKey").asString(); String sessionToken = cursor.field("sessionToken").asString(); - return new BasicSessionCredentials(accessKey, secretKey, sessionToken); + Instant defaultExpiry = Instant.now().plus(Duration.ofHours(1)); + Instant expiry; + try { + expiry = SlimeUtils.optionalString(cursor.field("expiry")).map(Instant::parse).orElse(defaultExpiry); + } catch (Exception e) { + expiry = defaultExpiry; + logger.warning("Unable to read expiry from credentials"); + } + return new Credentials(accessKey, secretKey, sessionToken, expiry); } catch (IOException e) { throw new UncheckedIOException(e); } } + + record Credentials (String awsAccessKey, String awsSecretKey, String sessionToken, Instant expiry) implements AWSSessionCredentials { + @Override + public String getSessionToken() { + return sessionToken; + } + + @Override + public String getAWSAccessKeyId() { + return awsAccessKey; + } + + @Override + public String getAWSSecretKey() { + return awsSecretKey; + } + } } diff --git a/jdisc-cloud-aws/src/test/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProviderTest.java b/jdisc-cloud-aws/src/test/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProviderTest.java new file mode 100644 index 00000000000..63cfd2f1eeb --- /dev/null +++ b/jdisc-cloud-aws/src/test/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProviderTest.java @@ -0,0 +1,66 @@ +package com.yahoo.jdisc.cloud.aws; + +import com.amazonaws.auth.AWSCredentials; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; + + +public class VespaAwsCredentialsProviderTest { + Path credentialsPath = TestFileSystem.create().getPath("/credentials.json"); + ManualClock clock = new ManualClock(Instant.now()); + + @Test + void refreshes_credentials() throws IOException { + Instant originalExpiry = clock.instant().plus(Duration.ofHours(12)); + writeCredentials(credentialsPath, originalExpiry); + VespaAwsCredentialsProvider credentialsProvider = new VespaAwsCredentialsProvider(credentialsPath, clock); + AWSCredentials credentials = credentialsProvider.getCredentials(); + assertExpiryEquals(originalExpiry, credentials); + + Instant updatedExpiry = clock.instant().plus(Duration.ofHours(24)); + writeCredentials(credentialsPath, updatedExpiry); + // File updated, but old credentials still valid + credentials = credentialsProvider.getCredentials(); + assertExpiryEquals(originalExpiry, credentials); + + // Credentials refreshes when it is < 30 minutes left until expiry + clock.advance(Duration.ofHours(11).plus(Duration.ofMinutes(31))); + credentials = credentialsProvider.getCredentials(); + assertExpiryEquals(updatedExpiry, credentials); + } + + @Test + void deserializes_credentials() throws IOException { + Instant originalExpiry = clock.instant().plus(Duration.ofHours(12)); + writeCredentials(credentialsPath, originalExpiry); + VespaAwsCredentialsProvider credentialsProvider = new VespaAwsCredentialsProvider(credentialsPath, clock); + AWSCredentials credentials = credentialsProvider.getCredentials(); + assertExpiryEquals(originalExpiry, credentials); + Assertions.assertEquals("awsAccessKey", credentials.getAWSAccessKeyId()); + Assertions.assertEquals("awsSecretKey", credentials.getAWSSecretKey()); + Assertions.assertEquals("sessionToken", ((VespaAwsCredentialsProvider.Credentials)credentials).getSessionToken()); + } + + private void writeCredentials(Path path, Instant expiry) throws IOException { + String content = """ + { + "awsAccessKey": "awsAccessKey", + "awsSecretKey": "awsSecretKey", + "sessionToken": "sessionToken", + "expiry": "%s" + }""".formatted(expiry.toString()); + Files.writeString(path, content); + } + + private void assertExpiryEquals(Instant expiry, AWSCredentials credentials) { + Assertions.assertEquals(expiry, ((VespaAwsCredentialsProvider.Credentials)credentials).expiry()); + } +} diff --git a/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java b/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java index 5ad6a382abd..f0439a21fec 100644 --- a/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java +++ b/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java @@ -26,6 +26,7 @@ public class LinguisticsCase { public static String toLowerCase(String in) { // def is picked from http://docs.oracle.com/javase/6/docs/api/java/lang/String.html#toLowerCase%28%29 // Also, at the time of writing, English is the default language for queries + if (in == null) return null; return Lowercase.toLowerCase(in); } diff --git a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java index 33f5ee7e4bb..9178c2d7e09 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java +++ b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java @@ -189,9 +189,8 @@ public class GramSplitter { @Override public boolean equals(Object o) { if (this == o) return true; - if ( ! (o instanceof Gram)) return false; + if ( ! (o instanceof Gram gram)) return false; - Gram gram = (Gram)o; if (codePointCount != gram.codePointCount) return false; if (start != gram.start) return false; return true; diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java index 6cc68c7ac14..809e9b8d133 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java @@ -15,35 +15,48 @@ import java.util.Objects; public class SimpleToken implements Token { private final List<Token> components = new ArrayList<>(); - private final String orig; + private final String original; private TokenType type = TokenType.UNKNOWN; private TokenScript script = TokenScript.UNKNOWN; private String tokenString; + private List<String> stems = null; // Any additional stems after tokenString private boolean specialToken = false; private long offset = 0; - public SimpleToken(String orig) { - this(orig, null); + public SimpleToken(String original) { + this(original, (String)null); } - public SimpleToken(String orig, String tokenString) { - this.orig = orig; + public SimpleToken(String original, String tokenString) { + this.original = original; this.tokenString = tokenString; } + /** Exposed as fromStems */ + private SimpleToken(String original, List<String> stems) { + this.type = TokenType.ALPHABETIC; // Only type which may have stems + this.original = original; + this.tokenString = stems.get(0); + this.stems = List.copyOf(stems.subList(1, stems.size())); + } + @Override public String getOrig() { - return orig; + return original; } @Override public int getNumStems() { - return tokenString != null ? 1 : 0; + return (tokenString != null ? 1 : 0) + (stems != null ? stems.size() : 0); } @Override public String getStem(int i) { - return tokenString; + if (i == 0) + return tokenString; + if (stems != null && i-1 < stems.size()) + return stems.get(i-1); + return tokenString; // TODO Vespa 9: throw new IllegalArgumentException() instead } @Override @@ -131,12 +144,12 @@ public class SimpleToken implements Token { @Override public int hashCode() { - return orig.hashCode(); + return original.hashCode(); } @Override public String toString() { - return "token '" + orig + "'"; + return "token '" + original + "'"; } public String toDetailString() { @@ -171,4 +184,8 @@ public class SimpleToken implements Token { return getType().isIndexable() && (getOrig().length() > 0); } + public static SimpleToken fromStems(String original, List<String> stems) { + return new SimpleToken(original, stems); + } + } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index 98a84a48095..b72d2bd6d37 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -106,7 +106,7 @@ public class SimpleTokenizer implements Tokenizer { String oldToken = token; token = stemmer.stem(token); String newToken = token; - log.log(Level.FINEST, () -> "stem '" + oldToken+"' to '" + newToken+"'"); + log.log(Level.FINEST, () -> "stem '" + oldToken + "' to '" + newToken + "'"); } String result = token; log.log(Level.FINEST, () -> "processed token is: " + result); diff --git a/parent/pom.xml b/parent/pom.xml index 15aee603f17..7d64988d5a5 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -237,7 +237,7 @@ <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>exec-maven-plugin</artifactId> - <version>3.1.0</version> + <version>3.1.1</version> </plugin> <plugin> <groupId>org.codehaus.mojo</groupId> @@ -317,7 +317,7 @@ --> <groupId>org.openrewrite.maven</groupId> <artifactId>rewrite-maven-plugin</artifactId> - <version>5.11.0</version> + <version>5.12.0</version> <configuration> <activeRecipes> <recipe>org.openrewrite.java.testing.junit5.JUnit5BestPractices</recipe> diff --git a/screwdriver.yaml b/screwdriver.yaml index c2b2572dd99..d7ddaa853f4 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -6,7 +6,7 @@ cache: component-build: [/main_job_cache] shared: - image: vespaengine/vespa-build-centos-stream8:latest + image: docker.io/vespaengine/vespa-build-almalinux-8 environment: USER_SHELL_BIN: bash annotations: @@ -199,13 +199,13 @@ jobs: fi docker build --file Dockerfile.systemtest \ --progress=plain \ - --build-arg VESPA_BASE_IMAGE=vespaengine/vespa-systemtest-base-centos-stream8:latest \ + --build-arg VESPA_BASE_IMAGE=docker.io/vespaengine/vespa-systemtest-base-el8:latest \ --build-arg SYSTEMTEST_BASE_IMAGE=vespa --build-arg SKIP_M2_POPULATE=$SKIP_M2_POPULATE \ --target systemtest \ - --tag vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION \ - --tag vespaengine/vespa-systemtest-centos-stream8:latest . + --tag docker.io/vespaengine/vespa-systemtest-el8:$VESPA_VERSION \ + --tag docker.io/vespaengine/vespa-systemtest-el8:latest . - verify-test-image: | - docker run --rm -ti --entrypoint bash vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION -lc \ + docker run --rm -ti --entrypoint bash docker.io/vespaengine/vespa-systemtest-el8:$VESPA_VERSION -lc \ "/opt/vespa-systemtests/lib/node_server.rb & sleep 3 && ruby /opt/vespa-systemtests/tests/search/basicsearch/basic_search.rb --run test_basicsearch__ELASTIC" - publish-test-image: | if [[ -z "$SD_PULL_REQUEST" ]]; then @@ -213,8 +213,8 @@ jobs: set +x docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" eval "$OPT_STATE" - docker push docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION - docker push docker.io/vespaengine/vespa-systemtest-centos-stream8:latest + docker push docker.io/vespaengine/vespa-systemtest-el8:$VESPA_VERSION + docker push docker.io/vespaengine/vespa-systemtest-el8:latest fi - *save-cache - update-build-status: | @@ -233,8 +233,6 @@ jobs: fi publish-release: - image: docker.io/vespaengine/vespa-build-centos-stream8:latest - annotations: screwdriver.cd/cpu: 7 screwdriver.cd/ram: 16 @@ -278,7 +276,7 @@ jobs: meta set vespa.version $VESPA_VERSION - install-dependencies: | dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - dnf install -y docker-ce docker-ce-cli containerd.io + dnf install -y docker-ce docker-ce-cli containerd.io python3 python3-pip docker system info - release-java-artifacts: | screwdriver/release-java-artifacts.sh $VESPA_VERSION $VESPA_REF @@ -294,8 +292,6 @@ jobs: screwdriver/factory-command.sh update-released-time $VESPA_VERSION publish-legacy-release: - image: docker.io/vespaengine/vespa-build-centos-stream8:latest - annotations: screwdriver.cd/cpu: 7 screwdriver.cd/ram: 16 @@ -307,7 +303,7 @@ jobs: screwdriver.cd/buildPeriodically: H 6 1 * * environment: - IMAGE_NAME: "vespaengine/vespa-generic-intel-x86_64" + IMAGE_NAME: "docker.io/vespaengine/vespa-generic-intel-x86_64" secrets: - DOCKER_HUB_DEPLOY_KEY @@ -374,7 +370,6 @@ jobs: fi publish-el9-preview: - image: docker.io/vespaengine/vespa-build-centos-stream8:latest annotations: screwdriver.cd/cpu: 7 screwdriver.cd/ram: 16 @@ -422,8 +417,8 @@ jobs: --build-arg VESPA_BASE_IMAGE=$BASE_IMAGE \ --build-arg VESPA_VERSION=$VESPA_VERSION \ --file Dockerfile \ - --tag vespaengine/vespa:latest \ - --tag vespaengine/$IMAGE_NAME:latest \ + --tag docker.io/vespaengine/vespa:latest \ + --tag docker.io/vespaengine/$IMAGE_NAME:latest \ . - verify-container-image: | # Run quick start guide @@ -452,7 +447,6 @@ jobs: fi publish-el8-preview: - image: docker.io/vespaengine/vespa-build-centos-stream8:latest annotations: screwdriver.cd/cpu: 7 screwdriver.cd/ram: 16 @@ -517,8 +511,7 @@ jobs: fi verify-opensource-rpm-installable: - requires: [~pr] - image: quay.io/centos/centos:stream8 + image: docker.io/almalinux:8 annotations: screwdriver.cd/buildPeriodically: H 0 * * * steps: @@ -532,7 +525,7 @@ jobs: mirror-copr-rpms-to-archive: requires: [publish-release] - image: quay.io/centos/centos:stream8 + image: docker.io/almalinux:8 annotations: screwdriver.cd/cpu: LOW screwdriver.cd/ram: LOW @@ -562,7 +555,7 @@ jobs: screwdriver/delete-old-cloudsmith-artifacts.sh mirror-copr-rpms-to-artifactory: - image: quay.io/centos/centos:stream8 + image: docker.io/almalinux:8 annotations: screwdriver.cd/cpu: LOW screwdriver.cd/ram: LOW diff --git a/screwdriver/test-quick-start-guide.sh b/screwdriver/test-quick-start-guide.sh index eead39fecaa..4c3feb35a21 100755 --- a/screwdriver/test-quick-start-guide.sh +++ b/screwdriver/test-quick-start-guide.sh @@ -13,7 +13,7 @@ cd $TESTDIR # Clone and setup doc tests git clone -q --depth 1 https://github.com/vespa-engine/documentation cd documentation -python3.9 -m pip install -qqq -r test/requirements.txt --user +python3 -m pip install -qqq -r test/requirements.txt --user echo -e "urls:\n - en/vespa-quick-start.html" > test/_quick-start.yaml # Get the required vespa CLI @@ -22,5 +22,5 @@ curl -fsSL https://github.com/vespa-engine/vespa/releases/download/v${VESPA_CLI_ ln -sf /opt/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64/bin/vespa /usr/local/bin/ # Run test -python3.9 test/test.py -v -c test/_quick-start.yaml +python3 test/test.py -v -c test/_quick-start.yaml diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index a7b45feb043..c0dfc0e1d29 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -257,6 +257,9 @@ public class RankingExpression implements Serializable { * @return a list of named rank properties required to implement this expression */ public Map<String, String> getRankProperties(SerializationContext context) { + if ("".equals(name)) { + return Map.of(); + } Deque<String> path = new LinkedList<>(); String serializedRoot = root.toString(new StringBuilder(), context, path, null).toString(); Map<String, String> serializedExpressions = context.serializedFunctions(); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java index 8af77ec1cdd..8d60f893c7c 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java @@ -364,7 +364,7 @@ public class RankingExpressionTestCase { private void assertSerialization(String expectedSerialization, String expressionString) { String serializedExpression; try { - RankingExpression expression = new RankingExpression(expressionString); + RankingExpression expression = new RankingExpression("secondphase", expressionString); // No functions -> expect one rank property serializedExpression = expression.getRankProperties(new SerializationContext()).values().iterator().next(); assertEquals(expectedSerialization, serializedExpression); @@ -376,7 +376,7 @@ public class RankingExpressionTestCase { try { // No functions -> output should be parseable to a ranking expression // (but not the same one due to primitivization) - RankingExpression reparsedExpression = new RankingExpression(serializedExpression); + RankingExpression reparsedExpression = new RankingExpression("secondphase", serializedExpression); // Serializing the primitivized expression should yield the same expression again String reserializedExpression = reparsedExpression.getRankProperties(new SerializationContext()).values().iterator().next(); @@ -399,7 +399,7 @@ public class RankingExpressionTestCase { if (print) System.out.println("Parsing expression '" + expressionString + "':"); - RankingExpression expression = new RankingExpression(expressionString); + RankingExpression expression = new RankingExpression("secondphase", expressionString); Map<String, String> rankProperties = expression.getRankProperties(new SerializationContext(functions, Optional.empty())); if (print) { diff --git a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp index 63c0b784018..dfea4901180 100644 --- a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp @@ -43,9 +43,8 @@ struct BitVectorTest { using AttributePtr = AttributeVector::SP; - BitVectorTest() { } - - ~BitVectorTest() { } + BitVectorTest(); + ~BitVectorTest(); template <typename VectorType> VectorType & as(AttributePtr &v); @@ -102,6 +101,9 @@ struct BitVectorTest test(BasicType bt, CollectionType ct, const vespalib::string &pref); }; +BitVectorTest::BitVectorTest() = default; +BitVectorTest::~BitVectorTest() = default; + template <typename VectorType> VectorType & @@ -427,16 +429,14 @@ BitVectorTest::test(BasicType bt, CollectionType ct, const vespalib::string &pre SearchContextPtr sc = getSearch<VectorType>(tv, true); checkSearch(v, std::move(sc), 2, 1022, 205, !fastSearch && !filter, true); - sc = getSearch<VectorType>(tv, false); + sc = getSearch<VectorType>(tv, filter); checkSearch(v, std::move(sc), 2, 1022, 205, !filter, true); const search::IDocumentWeightAttribute *dwa = v->asDocumentWeightAttribute(); if (dwa != nullptr) { - search::IDocumentWeightAttribute::LookupResult lres = - dwa->lookup(getSearchStr<VectorType>(), dwa->get_dictionary_snapshot()); + auto lres = dwa->lookup(getSearchStr<VectorType>(), dwa->get_dictionary_snapshot()); using DWSI = search::queryeval::DocumentWeightSearchIterator; - using SI = search::queryeval::SearchIterator; TermFieldMatchData md; - SI::UP dwsi(new DWSI(md, *dwa, lres)); + auto dwsi = std::make_unique<DWSI>(md, *dwa, lres); if (!filter) { TEST_DO(checkSearch(v, std::move(dwsi), md, 2, 1022, 205, !filter, true)); } else { @@ -445,13 +445,13 @@ BitVectorTest::test(BasicType bt, CollectionType ct, const vespalib::string &pre } } populate(tv, 2, 973, false); - sc = getSearch<VectorType>(tv, true); + sc = getSearch<VectorType>(tv, filter); checkSearch(v, std::move(sc), 977, 1022, 10, !filter, true); populate(tv, 2, 973, true); sc = getSearch<VectorType>(tv, true); checkSearch(v, std::move(sc), 2, 1022, 205, !fastSearch && !filter, true); addDocs(v, 15000); - sc = getSearch<VectorType>(tv, true); + sc = getSearch<VectorType>(tv, filter); checkSearch(v, std::move(sc), 2, 1022, 205, !filter, true); populateAll(tv, 10, 15000, true); sc = getSearch<VectorType>(tv, true); diff --git a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp index 61d94e120f5..af61301c06f 100644 --- a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp +++ b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp @@ -96,7 +96,7 @@ protected: return sequence; } - void populate(uint32_t sequence_length); + void populate(std::vector<uint32_t> sequence_lengths, std::optional<std::function<void()>> clear_callback = std::nullopt); EntryRef get_posting_ref(int key); void test_compact_btree_nodes(uint32_t sequence_length); void test_compact_sequence(uint32_t sequence_length); @@ -120,21 +120,28 @@ PostingStoreTest::~PostingStoreTest() } void -PostingStoreTest::populate(uint32_t sequence_length) +PostingStoreTest::populate(std::vector<uint32_t> sequence_lengths, std::optional<std::function<void()>> clear_callback) { auto& store = _store; auto& dictionary = _value_store.get_dictionary(); std::vector<EntryRef> refs; - for (int i = 0; i < 9000; ++i) { - refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length)); + for (auto sequence_length : sequence_lengths) { + for (int i = 0; i < 9000; ++i) { + refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length)); + } } - dictionary.update_posting_list(_value_store.insert(1), _value_store.get_comparator(), [this, sequence_length](EntryRef) { return add_sequence(4, 4 + sequence_length); }); - dictionary.update_posting_list(_value_store.insert(2), _value_store.get_comparator(), [this, sequence_length](EntryRef) { return add_sequence(5, 5 + sequence_length); }); - for (int i = 9000; i < 11000; ++i) { - refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length)); + dictionary.update_posting_list(_value_store.insert(1), _value_store.get_comparator(), [this, sequence_length = sequence_lengths.front()](EntryRef) { return add_sequence(4, 4 + sequence_length); }); + dictionary.update_posting_list(_value_store.insert(2), _value_store.get_comparator(), [this, sequence_length = sequence_lengths.front()](EntryRef) { return add_sequence(5, 5 + sequence_length); }); + for (auto sequence_length : sequence_lengths) { + for (int i = 9000; i < 11000; ++i) { + refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length)); + } } for (auto& ref : refs) { store.clear(ref); + if (clear_callback.has_value()) { + clear_callback.value()(); + } } inc_generation(); } @@ -150,7 +157,7 @@ PostingStoreTest::get_posting_ref(int key) void PostingStoreTest::test_compact_sequence(uint32_t sequence_length) { - populate(sequence_length); + populate({sequence_length}); auto &store = _store; EntryRef old_ref1 = get_posting_ref(1); EntryRef old_ref2 = get_posting_ref(2); @@ -183,7 +190,7 @@ PostingStoreTest::test_compact_sequence(uint32_t sequence_length) void PostingStoreTest::test_compact_btree_nodes(uint32_t sequence_length) { - populate(sequence_length); + populate({sequence_length}); auto &store = _store; EntryRef old_ref1 = get_posting_ref(1); EntryRef old_ref2 = get_posting_ref(2); @@ -246,6 +253,40 @@ TEST_P(PostingStoreTest, require_that_bitvectors_are_compacted) test_compact_sequence(huge_sequence_length); } +namespace { + +/* + * Check if compaction of btree nodes or short arrays is suppressed due to + * dead ratio being too low for the sum of both data stores. + */ +bool compaction_is_suppressed(MyPostingStore& store, CompactionStrategy& compaction_strategy) +{ + store.update_stat(compaction_strategy); + auto& compaction_spec = store.get_compaction_spec(); + auto memory_usage = store.getMemoryUsage(); + return ((compaction_strategy.should_compact_memory(memory_usage.btrees) && + !compaction_spec.btree_nodes()) || + (compaction_strategy.should_compact_memory(memory_usage.short_arrays) && + !compaction_spec.store())); +} + +} + +TEST_P(PostingStoreTest, require_that_compaction_is_suppressed) +{ + CompactionStrategy compaction_strategy(0.05, 0.2); + bool suppressed_compaction = false; + auto clear_callback = [this, &compaction_strategy, &suppressed_compaction]() mutable + { + inc_generation(); + if (compaction_is_suppressed(_store, compaction_strategy)) { + suppressed_compaction = true; + } + }; + populate({ 4, 10}, clear_callback); + EXPECT_TRUE(suppressed_compaction); +} + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp index 1d908bed568..372b76fa9af 100644 --- a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp @@ -81,7 +81,7 @@ public: } FieldSpecBaseList exposeFields() const override { - return FieldSpecBaseList(); + return {}; } bool inheritStrict(size_t i) const override { @@ -140,7 +140,7 @@ struct MyTerm : SimpleLeafBlueprint { setEstimate(HitEstimate(hitEstimate, false)); } SearchIterator::UP createLeafSearch(const search::fef::TermFieldMatchDataArray &, bool) const override { - return SearchIterator::UP(); + return {}; } SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override { return create_default_filter(strict, constraint); @@ -311,13 +311,13 @@ TEST("testHitEstimateCalculation") TEST("testHitEstimatePropagation") { - MyLeaf *leaf1 = new MyLeaf(); + auto *leaf1 = new MyLeaf(); leaf1->estimate(10); - MyLeaf *leaf2 = new MyLeaf(); + auto *leaf2 = new MyLeaf(); leaf2->estimate(20); - MyLeaf *leaf3 = new MyLeaf(); + auto *leaf3 = new MyLeaf(); leaf3->estimate(30); MyOr *parent = new MyOr(); @@ -346,7 +346,7 @@ TEST("testHitEstimatePropagation") leaf3->estimate(25); EXPECT_EQUAL(20u, root->getState().estimate().estHits); parent->addChild(std::move(tmp)); - EXPECT_TRUE(tmp.get() == 0); + EXPECT_FALSE(tmp); EXPECT_EQUAL(25u, root->getState().estimate().estHits); } @@ -572,7 +572,7 @@ TEST_F("testSearchCreation", Fixture) .addField(3, 3).create()); SearchIterator::UP leafsearch = f.create(*l); - MySearch *lw = new MySearch("leaf", true, true); + auto *lw = new MySearch("leaf", true, true); lw->addHandle(1).addHandle(2).addHandle(3); SearchIterator::UP wantleaf(lw); @@ -584,11 +584,11 @@ TEST_F("testSearchCreation", Fixture) .add(MyLeafSpec(2).addField(2, 2).create())); SearchIterator::UP andsearch = f.create(*a); - MySearch *l1 = new MySearch("leaf", true, true); - MySearch *l2 = new MySearch("leaf", true, false); + auto *l1 = new MySearch("leaf", true, true); + auto *l2 = new MySearch("leaf", true, false); l1->addHandle(1); l2->addHandle(2); - MySearch *aw = new MySearch("and", false, true); + auto *aw = new MySearch("and", false, true); aw->add(l1); aw->add(l2); SearchIterator::UP wanted(aw); @@ -600,11 +600,11 @@ TEST_F("testSearchCreation", Fixture) .add(MyLeafSpec(2).addField(2, 22).create())); SearchIterator::UP orsearch = f.create(*o); - MySearch *l1 = new MySearch("leaf", true, true); - MySearch *l2 = new MySearch("leaf", true, true); + auto *l1 = new MySearch("leaf", true, true); + auto *l2 = new MySearch("leaf", true, true); l1->addHandle(11); l2->addHandle(22); - MySearch *ow = new MySearch("or", false, true); + auto *ow = new MySearch("or", false, true); ow->add(l1); ow->add(l2); SearchIterator::UP wanted(ow); @@ -619,7 +619,7 @@ TEST("testBlueprintMakeNew") .add(MyLeafSpec(2).addField(2, 22).create())); orig->setSourceId(42); MyOr *myOr = dynamic_cast<MyOr*>(orig.get()); - ASSERT_TRUE(myOr != 0); + ASSERT_TRUE(myOr != nullptr); EXPECT_EQUAL(42u, orig->getSourceId()); EXPECT_EQUAL(2u, orig->getState().numFields()); } diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index a663944938c..b150f8db7c5 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -8,7 +8,6 @@ #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/equiv_blueprint.h> #include <vespa/searchlib/queryeval/multisearch.h> -#include <vespa/searchlib/queryeval/andnotsearch.h> #include <vespa/searchlib/queryeval/wand/weak_and_search.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/test/diskindex/testdiskindex.h> @@ -74,13 +73,13 @@ TEST("test AndNot Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); } @@ -125,7 +124,7 @@ TEST("test And propagates updated histestimate") { bp.fetchPostings(ExecuteInfo::TRUE); EXPECT_EQUAL(3u, bp.childCnt()); for (uint32_t i = 0; i < bp.childCnt(); i++) { - const RememberExecuteInfo & child = dynamic_cast<const RememberExecuteInfo &>(bp.getChild(i)); + const auto & child = dynamic_cast<const RememberExecuteInfo &>(bp.getChild(i)); EXPECT_EQUAL((i == 0), child.executeInfo.isStrict()); } EXPECT_EQUAL(1.0f, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(0)).executeInfo.hitRate()); @@ -133,22 +132,43 @@ TEST("test And propagates updated histestimate") { EXPECT_EQUAL(1.0f/(250*25), dynamic_cast<const RememberExecuteInfo &>(bp.getChild(2)).executeInfo.hitRate()); } +TEST("test Or propagates updated histestimate") { + OrBlueprint bp; + bp.setSourceId(2); + bp.addChild(ap(MyLeafSpec(5000).create<RememberExecuteInfo>()->setSourceId(2))); + bp.addChild(ap(MyLeafSpec(2000).create<RememberExecuteInfo>()->setSourceId(2))); + bp.addChild(ap(MyLeafSpec(800).create<RememberExecuteInfo>()->setSourceId(2))); + bp.addChild(ap(MyLeafSpec(20).create<RememberExecuteInfo>()->setSourceId(2))); + bp.optimize_self(); + bp.setDocIdLimit(5000); + bp.fetchPostings(ExecuteInfo::TRUE); + EXPECT_EQUAL(4u, bp.childCnt()); + for (uint32_t i = 0; i < bp.childCnt(); i++) { + const auto & child = dynamic_cast<const RememberExecuteInfo &>(bp.getChild(i)); + EXPECT_TRUE(child.executeInfo.isStrict()); + } + EXPECT_EQUAL(1.0f, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(0)).executeInfo.hitRate()); + EXPECT_EQUAL(1.0f, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(1)).executeInfo.hitRate()); + EXPECT_EQUAL(3.0f/5.0f, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(2)).executeInfo.hitRate()); + EXPECT_EQUAL(3.0f*42.0f/(5.0f*50.0f), dynamic_cast<const RememberExecuteInfo &>(bp.getChild(3)).executeInfo.hitRate()); +} + TEST("test And Blueprint") { AndBlueprint b; { // combine std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(5u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); } @@ -187,16 +207,16 @@ TEST("test Or Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); } @@ -259,16 +279,16 @@ TEST("test Near Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(5u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); } @@ -300,16 +320,16 @@ TEST("test ONear Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(5u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); } @@ -341,16 +361,16 @@ TEST("test Rank Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); } @@ -391,16 +411,16 @@ TEST("test SourceBlender Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); } @@ -453,60 +473,60 @@ TEST("test SourceBlender below AND optimization") { auto selector_1 = std::make_unique<InvalidSelector>(); // the one auto selector_2 = std::make_unique<InvalidSelector>(); // not the one //------------------------------------------------------------------------- - AndBlueprint *top = new AndBlueprint(); + auto *top = new AndBlueprint(); Blueprint::UP top_bp(top); top->addChild(ap(MyLeafSpec(2).create())); top->addChild(ap(MyLeafSpec(1).create())); top->addChild(ap(MyLeafSpec(3).create())); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); top->addChild(ap(blender)); } //------------------------------------------------------------------------- - AndBlueprint *expect = new AndBlueprint(); + auto *expect = new AndBlueprint(); Blueprint::UP expect_bp(expect); expect->addChild(ap(MyLeafSpec(1).create())); expect->addChild(ap(MyLeafSpec(2).create())); expect->addChild(ap(MyLeafSpec(3).create())); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); expect->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender(new SourceBlenderBlueprint(*selector_1)); + auto *blender(new SourceBlenderBlueprint(*selector_1)); { - AndBlueprint *sub_and = new AndBlueprint(); + auto *sub_and = new AndBlueprint(); sub_and->setSourceId(3); sub_and->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); sub_and->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); blender->addChild(ap(sub_and)); } { - AndBlueprint *sub_and = new AndBlueprint(); + auto *sub_and = new AndBlueprint(); sub_and->setSourceId(2); sub_and->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); sub_and->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); @@ -514,7 +534,7 @@ TEST("test SourceBlender below AND optimization") { blender->addChild(ap(sub_and)); } { - AndBlueprint *sub_and = new AndBlueprint(); + auto *sub_and = new AndBlueprint(); sub_and->setSourceId(1); sub_and->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); sub_and->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); @@ -535,51 +555,51 @@ TEST("test SourceBlender below OR optimization") { auto selector_1 = std::make_unique<InvalidSelector>(); // the one auto selector_2 = std::make_unique<InvalidSelector>(); // not the one //------------------------------------------------------------------------- - OrBlueprint *top = new OrBlueprint(); + auto *top = new OrBlueprint(); Blueprint::UP top_up(top); top->addChild(ap(MyLeafSpec(2).create())); top->addChild(ap(MyLeafSpec(1).create())); top->addChild(ap(MyLeafSpec(3).create())); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); top->addChild(ap(blender)); } //------------------------------------------------------------------------- - OrBlueprint *expect = new OrBlueprint(); + auto *expect = new OrBlueprint(); Blueprint::UP expect_up(expect); { - SourceBlenderBlueprint *blender(new SourceBlenderBlueprint(*selector_1)); + auto *blender(new SourceBlenderBlueprint(*selector_1)); { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(3); sub_and->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); sub_and->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); blender->addChild(ap(sub_and)); } { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(2); sub_and->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); sub_and->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); @@ -587,7 +607,7 @@ TEST("test SourceBlender below OR optimization") { blender->addChild(ap(sub_and)); } { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(1); sub_and->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); sub_and->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); @@ -597,7 +617,7 @@ TEST("test SourceBlender below OR optimization") { expect->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); expect->addChild(ap(blender)); @@ -617,10 +637,10 @@ TEST("test SourceBlender below AND_NOT optimization") { auto selector_1 = std::make_unique<InvalidSelector>(); // the one auto selector_2 = std::make_unique<InvalidSelector>(); // not the one //------------------------------------------------------------------------- - AndNotBlueprint *top = new AndNotBlueprint(); + auto *top = new AndNotBlueprint(); Blueprint::UP top_up(top); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(42).create()->setSourceId(1))); top->addChild(ap(blender)); } @@ -628,50 +648,50 @@ TEST("test SourceBlender below AND_NOT optimization") { top->addChild(ap(MyLeafSpec(1).create())); top->addChild(ap(MyLeafSpec(3).create())); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); top->addChild(ap(blender)); } //------------------------------------------------------------------------- - AndNotBlueprint *expect = new AndNotBlueprint(); + auto *expect = new AndNotBlueprint(); Blueprint::UP expect_up(expect); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(42).create()->setSourceId(1))); expect->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender(new SourceBlenderBlueprint(*selector_1)); + auto *blender(new SourceBlenderBlueprint(*selector_1)); { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(3); sub_and->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); sub_and->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); blender->addChild(ap(sub_and)); } { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(2); sub_and->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); sub_and->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); @@ -679,7 +699,7 @@ TEST("test SourceBlender below AND_NOT optimization") { blender->addChild(ap(sub_and)); } { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(1); sub_and->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); sub_and->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); @@ -689,7 +709,7 @@ TEST("test SourceBlender below AND_NOT optimization") { expect->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); expect->addChild(ap(blender)); @@ -709,10 +729,10 @@ TEST("test SourceBlender below RANK optimization") { auto selector_1 = std::make_unique<InvalidSelector>(); // the one auto selector_2 = std::make_unique<InvalidSelector>(); // not the one //------------------------------------------------------------------------- - RankBlueprint *top = new RankBlueprint(); + auto *top = new RankBlueprint(); Blueprint::UP top_up(top); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(42).create()->setSourceId(1))); top->addChild(ap(blender)); } @@ -720,36 +740,36 @@ TEST("test SourceBlender below RANK optimization") { top->addChild(ap(MyLeafSpec(1).create())); top->addChild(ap(MyLeafSpec(3).create())); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); top->addChild(ap(blender)); } //------------------------------------------------------------------------- - RankBlueprint *expect = new RankBlueprint(); + auto *expect = new RankBlueprint(); Blueprint::UP expect_up(expect); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(42).create()->setSourceId(1))); expect->addChild(ap(blender)); } @@ -757,22 +777,22 @@ TEST("test SourceBlender below RANK optimization") { expect->addChild(ap(MyLeafSpec(1).create())); expect->addChild(ap(MyLeafSpec(3).create())); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); expect->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender(new SourceBlenderBlueprint(*selector_1)); + auto *blender(new SourceBlenderBlueprint(*selector_1)); { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(3); sub_and->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); sub_and->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); blender->addChild(ap(sub_and)); } { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(2); sub_and->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); sub_and->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); @@ -780,7 +800,7 @@ TEST("test SourceBlender below RANK optimization") { blender->addChild(ap(sub_and)); } { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(1); sub_and->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); sub_and->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); @@ -826,7 +846,7 @@ TEST("test empty root node optimization and safeness") { addChild(ap(MyLeafSpec(0, true).create())). addChild(ap(MyLeafSpec(0, true).create())))); //------------------------------------------------------------------------- - Blueprint::UP expect_up(new EmptyBlueprint()); + auto expect_up = std::make_unique<EmptyBlueprint>(); //------------------------------------------------------------------------- top1_up = Blueprint::optimize(std::move(top1_up)); top2_up = Blueprint::optimize(std::move(top2_up)); @@ -851,7 +871,7 @@ TEST("and with one empty child is optimized away") { top = Blueprint::optimize(std::move(top)); Blueprint::UP expect_up(ap((new SourceBlenderBlueprint(*selector))-> addChild(ap(MyLeafSpec(10).create())). - addChild(ap(new EmptyBlueprint())))); + addChild(std::make_unique<EmptyBlueprint>()))); EXPECT_EQUAL(expect_up->asString(), top->asString()); } @@ -964,11 +984,11 @@ TEST("require that replaced blueprints retain source id") { //------------------------------------------------------------------------- // replace empty root with empty search Blueprint::UP top1_up(ap(MyLeafSpec(0, true).create()->setSourceId(13))); - Blueprint::UP expect1_up(new EmptyBlueprint()); + auto expect1_up = std::make_unique<EmptyBlueprint>(); expect1_up->setSourceId(13); //------------------------------------------------------------------------- // replace self with single child - Blueprint::UP top2_up(ap(static_cast<AndBlueprint&>((new AndBlueprint())->setSourceId(42)). + Blueprint::UP top2_up(ap(dynamic_cast<AndBlueprint&>((new AndBlueprint())->setSourceId(42)). addChild(ap(MyLeafSpec(30).create()->setSourceId(55))))); Blueprint::UP expect2_up(ap(MyLeafSpec(30).create()->setSourceId(42))); //------------------------------------------------------------------------- @@ -1017,16 +1037,16 @@ TEST("test WeakAnd Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); } @@ -1057,14 +1077,14 @@ TEST("test WeakAnd Blueprint") { { WeakAndBlueprint wa(456); MatchData::UP md = MatchData::makeTestInstance(100, 10); - wa.addTerm(Blueprint::UP(new FakeBlueprint(field, x)), 120); - wa.addTerm(Blueprint::UP(new FakeBlueprint(field, z)), 140); - wa.addTerm(Blueprint::UP(new FakeBlueprint(field, y)), 130); + wa.addTerm(std::make_unique<FakeBlueprint>(field, x), 120); + wa.addTerm(std::make_unique<FakeBlueprint>(field, z), 140); + wa.addTerm(std::make_unique<FakeBlueprint>(field, y), 130); { wa.fetchPostings(ExecuteInfo::TRUE); SearchIterator::UP search = wa.createSearch(*md, true); - EXPECT_TRUE(dynamic_cast<WeakAndSearch*>(search.get()) != 0); - WeakAndSearch &s = dynamic_cast<WeakAndSearch&>(*search); + EXPECT_TRUE(dynamic_cast<WeakAndSearch*>(search.get()) != nullptr); + auto &s = dynamic_cast<WeakAndSearch&>(*search); EXPECT_EQUAL(456u, s.getN()); ASSERT_EQUAL(3u, s.getTerms().size()); EXPECT_GREATER(s.get_max_score(0), 0.0); @@ -1085,7 +1105,7 @@ TEST("test WeakAnd Blueprint") { { wa.fetchPostings(ExecuteInfo::FALSE); SearchIterator::UP search = wa.createSearch(*md, false); - EXPECT_TRUE(dynamic_cast<WeakAndSearch*>(search.get()) != 0); + EXPECT_TRUE(dynamic_cast<WeakAndSearch*>(search.get()) != nullptr); EXPECT_TRUE(search->seek(1)); EXPECT_TRUE(search->seek(2)); EXPECT_FALSE(search->seek(3)); @@ -1195,7 +1215,7 @@ namespace { SimpleStringTerm makeTerm(const std::string & term) { - return SimpleStringTerm(term, "field", 0, search::query::Weight(0)); + return {term, "field", 0, search::query::Weight(0)}; } } @@ -1231,7 +1251,7 @@ TEST("require that children does not optimize when parents refuse them to") { SearchIterator::UP search = top_up->createSearch(*md, true); EXPECT_EQUAL("search::queryeval::EquivImpl<true>", search->getClassName()); { - const MultiSearch & e = dynamic_cast<const MultiSearch &>(*search); + const auto & e = dynamic_cast<const MultiSearch &>(*search); EXPECT_EQUAL("search::BitVectorIteratorStrictT<false>", e.getChildren()[0]->getClassName()); EXPECT_EQUAL("search::diskindex::ZcRareWordPosOccIterator<true, false>", e.getChildren()[1]->getClassName()); EXPECT_EQUAL("search::diskindex::ZcRareWordPosOccIterator<true, false>", e.getChildren()[2]->getClassName()); @@ -1241,7 +1261,7 @@ TEST("require that children does not optimize when parents refuse them to") { search = top_up->createSearch(*md, true); EXPECT_EQUAL("search::queryeval::EquivImpl<true>", search->getClassName()); { - const MultiSearch & e = dynamic_cast<const MultiSearch &>(*search); + const auto & e = dynamic_cast<const MultiSearch &>(*search); EXPECT_EQUAL("search::BitVectorIteratorStrictT<false>", e.getChildren()[0]->getClassName()); EXPECT_EQUAL("search::diskindex::ZcRareWordPosOccIterator<true, false>", e.getChildren()[1]->getClassName()); EXPECT_EQUAL("search::diskindex::ZcRareWordPosOccIterator<true, false>", e.getChildren()[2]->getClassName()); @@ -1269,7 +1289,7 @@ TEST("require_that_unpack_optimization_is_not_overruled_by_equiv") { SearchIterator::UP search = top_up->createSearch(*md, true); EXPECT_EQUAL("search::queryeval::EquivImpl<true>", search->getClassName()); { - const MultiSearch & e = dynamic_cast<const MultiSearch &>(*search); + const auto & e = dynamic_cast<const MultiSearch &>(*search); EXPECT_EQUAL("search::queryeval::OrLikeSearch<true, search::queryeval::(anonymous namespace)::FullUnpack>", e.getChildren()[0]->getClassName()); } @@ -1278,7 +1298,7 @@ TEST("require_that_unpack_optimization_is_not_overruled_by_equiv") { search = top_up->createSearch(*md, true); EXPECT_EQUAL("search::queryeval::EquivImpl<true>", search->getClassName()); { - const MultiSearch & e = dynamic_cast<const MultiSearch &>(*search); + const auto & e = dynamic_cast<const MultiSearch &>(*search); EXPECT_EQUAL("search::queryeval::OrLikeSearch<true, search::queryeval::(anonymous namespace)::SelectiveUnpack>", e.getChildren()[0]->getClassName()); } @@ -1288,7 +1308,7 @@ TEST("require_that_unpack_optimization_is_not_overruled_by_equiv") { search = top_up->createSearch(*md, true); EXPECT_EQUAL("search::queryeval::EquivImpl<true>", search->getClassName()); { - const MultiSearch & e = dynamic_cast<const MultiSearch &>(*search); + const auto & e = dynamic_cast<const MultiSearch &>(*search); EXPECT_EQUAL("search::queryeval::OrLikeSearch<true, search::queryeval::NoUnpack>", e.getChildren()[0]->getClassName()); } @@ -1344,7 +1364,7 @@ TEST("require that children of onear are not optimized") { TEST("require that ANDNOT without children is optimized to empty search") { Blueprint::UP top_up(new AndNotBlueprint()); - Blueprint::UP expect_up(new EmptyBlueprint()); + auto expect_up = std::make_unique<EmptyBlueprint>(); top_up = Blueprint::optimize(std::move(top_up)); EXPECT_EQUAL(expect_up->asString(), top_up->asString()); } diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 8bee8b3f7f7..2e9e5470b5f 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -44,8 +44,8 @@ PostingStoreBase2::resizeBitVectors(uint32_t newSize, uint32_t newCapacity) newSize = newCapacity; if (newSize == _bvSize && newCapacity == _bvCapacity) return false; - _minBvDocFreq = std::max(newSize >> 6, 64u); - _maxBvDocFreq = std::max(newSize >> 5, 128u); + _minBvDocFreq = std::max(newSize >> 7, 64u); + _maxBvDocFreq = std::max(newSize >> 6, 128u); if (_bvs.empty()) { _bvSize = newSize; _bvCapacity = newCapacity; @@ -613,9 +613,13 @@ PostingStore<DataT>::update_stat(const CompactionStrategy& compaction_strategy) vespalib::MemoryUsage usage; auto btree_nodes_memory_usage = _allocator.getMemoryUsage(); auto store_memory_usage = _store.getMemoryUsage(); - _compaction_spec = PostingStoreCompactionSpec(compaction_strategy.should_compact_memory(btree_nodes_memory_usage), compaction_strategy.should_compact_memory(store_memory_usage)); usage.merge(btree_nodes_memory_usage); usage.merge(store_memory_usage); + if (compaction_strategy.should_compact_memory(usage)) { + _compaction_spec = PostingStoreCompactionSpec(compaction_strategy.should_compact_memory(btree_nodes_memory_usage), compaction_strategy.should_compact_memory(store_memory_usage)); + } else { + _compaction_spec = PostingStoreCompactionSpec(); + } uint64_t bvExtraBytes = _bvExtraBytes; usage.incUsedBytes(bvExtraBytes); usage.incAllocatedBytes(bvExtraBytes); diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h index e3fb88c81f1..bd19bbd3675 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.h +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h @@ -57,6 +57,9 @@ public: virtual ~PostingStoreBase2(); bool resizeBitVectors(uint32_t newSize, uint32_t newCapacity); virtual bool removeSparseBitVectors() = 0; + + // Only used by unit test. + const PostingStoreCompactionSpec& get_compaction_spec() const noexcept { return _compaction_spec; } }; template <typename DataT> diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index e006a95ae5f..aba80faaa8a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -117,7 +117,7 @@ Blueprint::State::State(FieldSpecBaseList fields_in) noexcept Blueprint::State::~State() = default; Blueprint::Blueprint() noexcept - : _parent(0), + : _parent(nullptr), _sourceId(0xffffffff), _docid_limit(0), _frozen(false) @@ -141,7 +141,7 @@ Blueprint::optimize_self() Blueprint::UP Blueprint::get_replacement() { - return Blueprint::UP(); + return {}; } void @@ -163,8 +163,8 @@ std::unique_ptr<MatchingElementsSearch> Blueprint::create_matching_elements_search(const MatchingElementsFields &fields) const { (void) fields; - return std::unique_ptr<MatchingElementsSearch>(); -}; + return {}; +} namespace { @@ -196,7 +196,7 @@ template <typename Op> std::unique_ptr<SearchIterator> create_op_filter(const Blueprint::Children &children, bool strict, Blueprint::FilterConstraint constraint) { - REQUIRE(children.size() > 0); + REQUIRE( ! children.empty()); MultiSearch::Children list; std::unique_ptr<SearchIterator> spare; list.reserve(children.size()); @@ -261,7 +261,7 @@ Blueprint::create_atmost_or_filter(const Children &children, bool strict, Bluepr std::unique_ptr<SearchIterator> Blueprint::create_andnot_filter(const Children &children, bool strict, Blueprint::FilterConstraint constraint) { - REQUIRE(children.size() > 0); + REQUIRE( ! children.empty() ); MultiSearch::Children list; list.reserve(children.size()); { @@ -291,7 +291,7 @@ Blueprint::create_andnot_filter(const Children &children, bool strict, Blueprint std::unique_ptr<SearchIterator> Blueprint::create_first_child_filter(const Children &children, bool strict, Blueprint::FilterConstraint constraint) { - REQUIRE(children.size() > 0); + REQUIRE(!children.empty()); return children[0]->createFilterSearch(strict, constraint); } @@ -434,7 +434,7 @@ IntermediateBlueprint::infer_allow_termwise_eval() const } } return true; -}; +} bool IntermediateBlueprint::infer_want_global_filter() const @@ -640,11 +640,7 @@ namespace { bool areAnyParentsEquiv(const Blueprint * node) { - return (node == nullptr) - ? false - : node->isEquiv() - ? true - : areAnyParentsEquiv(node->getParent()); + return (node != nullptr) && (node->isEquiv() || areAnyParentsEquiv(node->getParent())); } bool @@ -767,7 +763,7 @@ LeafBlueprint::set_tree_size(uint32_t value) void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const search::queryeval::Blueprint *obj) { - if (obj != 0) { + if (obj != nullptr) { self.openStruct(name, obj->getClassName()); obj->visitMembers(self); self.closeStruct(); diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index c9ebab7ae9d..da6050f075d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -79,8 +79,8 @@ public: static constexpr uint8_t COST_TIER_MAX = 255; State() noexcept; - State(FieldSpecBase field) noexcept; - State(FieldSpecBaseList fields_in) noexcept; + explicit State(FieldSpecBase field) noexcept; + explicit State(FieldSpecBaseList fields_in) noexcept; State(const State &rhs) = delete; State(State &&rhs) noexcept = default; State &operator=(const State &rhs) = delete; @@ -105,7 +105,7 @@ public: _estimateHits = est.estHits; _estimateEmpty = est.empty; } - HitEstimate estimate() const noexcept { return HitEstimate(_estimateHits, _estimateEmpty); } + HitEstimate estimate() const noexcept { return {_estimateHits, _estimateEmpty}; } double hit_ratio(uint32_t docid_limit) const noexcept { uint32_t total_hits = _estimateHits; uint32_t total_docs = std::max(total_hits, docid_limit); @@ -374,7 +374,7 @@ protected: void set_want_global_filter(bool value); void set_tree_size(uint32_t value); - LeafBlueprint(bool allow_termwise_eval) noexcept + explicit LeafBlueprint(bool allow_termwise_eval) noexcept : _state() { _state.allow_termwise_eval(allow_termwise_eval); diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp index 6e0bc695fe3..790d61b3731 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp @@ -263,6 +263,17 @@ AndBlueprint::computeNextHitRate(const Blueprint & child, double hitRate) const return hitRate * child.hit_ratio(); } +double +OrBlueprint::computeNextHitRate(const Blueprint & child, double hitRate) const { + // Avoid dropping hitRate to zero when meeting a conservatively high hitrate in a child. + // Happens at least when using non fast-search attributes, and with AND nodes. + constexpr double MIN_INVERSE_HIT_RATIO = 0.10; + double inverse_child_hit_ratio = 1.0 - child.hit_ratio(); + return (inverse_child_hit_ratio > MIN_INVERSE_HIT_RATIO) + ? hitRate * inverse_child_hit_ratio + : hitRate; +} + //----------------------------------------------------------------------------- OrBlueprint::~OrBlueprint() = default; @@ -596,7 +607,7 @@ RankBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) cons //----------------------------------------------------------------------------- -SourceBlenderBlueprint::SourceBlenderBlueprint(const ISourceSelector &selector) +SourceBlenderBlueprint::SourceBlenderBlueprint(const ISourceSelector &selector) noexcept : _selector(selector) { } @@ -626,27 +637,6 @@ SourceBlenderBlueprint::inheritStrict(size_t) const return true; } -class FindSource : public Blueprint::IPredicate -{ -public: - explicit FindSource(uint32_t sourceId) noexcept : _sourceId(sourceId) { } - bool check(const Blueprint & bp) const override { return bp.getSourceId() == _sourceId; } -private: - uint32_t _sourceId; -}; - -ssize_t -SourceBlenderBlueprint::findSource(uint32_t sourceId) const -{ - ssize_t index(-1); - FindSource fs(sourceId); - IndexList list = find(fs); - if ( ! list.empty()) { - index = list.front(); - } - return index; -} - SearchIterator::UP SourceBlenderBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData &) const diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h index b8ca46f7549..409a9e0fe95 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h @@ -74,6 +74,8 @@ public: bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; +private: + double computeNextHitRate(const Blueprint & child, double hitRate) const override; }; //----------------------------------------------------------------------------- @@ -95,8 +97,8 @@ public: bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; - WeakAndBlueprint(uint32_t n) : _n(n) {} - ~WeakAndBlueprint(); + explicit WeakAndBlueprint(uint32_t n) noexcept : _n(n) {} + ~WeakAndBlueprint() override; void addTerm(Blueprint::UP bp, uint32_t weight) { addChild(std::move(bp)); _weights.push_back(weight); @@ -124,7 +126,7 @@ public: bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; - NearBlueprint(uint32_t window) : _window(window) {} + explicit NearBlueprint(uint32_t window) noexcept : _window(window) {} }; //----------------------------------------------------------------------------- @@ -146,7 +148,7 @@ public: bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; - ONearBlueprint(uint32_t window) : _window(window) {} + explicit ONearBlueprint(uint32_t window) noexcept : _window(window) {} }; //----------------------------------------------------------------------------- @@ -176,18 +178,12 @@ private: const ISourceSelector &_selector; public: - SourceBlenderBlueprint(const ISourceSelector &selector); + explicit SourceBlenderBlueprint(const ISourceSelector &selector) noexcept; ~SourceBlenderBlueprint() override; HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; void sort(Children &children) const override; bool inheritStrict(size_t i) const override; - /** - * Will return the index matching the given sourceId. - * @param sourceId The sourceid to find. - * @return The index to the child representing the sourceId. -1 if not found. - */ - ssize_t findSource(uint32_t sourceId) const; SearchIterator::UP createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; diff --git a/security-utils/src/main/java/com/yahoo/security/Pkcs10CsrBuilder.java b/security-utils/src/main/java/com/yahoo/security/Pkcs10CsrBuilder.java index c34e3acb09f..629b610564a 100644 --- a/security-utils/src/main/java/com/yahoo/security/Pkcs10CsrBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/Pkcs10CsrBuilder.java @@ -90,7 +90,8 @@ public class Pkcs10CsrBuilder { .toArray(GeneralName[]::new)); extGen.addExtension(Extension.subjectAlternativeName, false, generalNames); } - requestBuilder.addAttribute(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extGen.generate()); + if (!extGen.isEmpty()) + requestBuilder.addAttribute(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extGen.generate()); ContentSigner contentSigner = new JcaContentSignerBuilder(signatureAlgorithm.getAlgorithmName()) .setProvider(BouncyCastleProviderHolder.getInstance()) .build(keyPair.getPrivate()); diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 61332bb9ad6..899c1979e86 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -122,9 +122,6 @@ void MetricsTest::createFakeLoad() metrics.docs.inc(10 * n); metrics.bytes.inc(10240 * n); } - _filestorMetrics->directoryEvents.inc(5); - _filestorMetrics->partitionEvents.inc(4); - _filestorMetrics->diskEvents.inc(3); { FileStorMetrics& disk(*_filestorMetrics); disk.queueSize.addValue(4 * n); @@ -147,18 +144,10 @@ void MetricsTest::createFakeLoad() thread.update.notFound.inc(1 * n); thread.update.latencyRead.addValue(2 * n); thread.update.latency.addValue(7 * n); - thread.revert.count.inc(2 * n); - thread.revert.notFound.inc(n / 2); - thread.revert.latency.addValue(2 * n); thread.visit.count.inc(6 * n); thread.deleteBuckets.count.inc(1 * n); - thread.repairs.count.inc(3 * n); - thread.repairFixed.inc(1 * n); thread.splitBuckets.count.inc(20 * n); - thread.movedBuckets.count.inc(1 * n); - thread.readBucketInfo.count.inc(2 * n); - thread.internalJoin.count.inc(3 * n); thread.mergeBuckets.count.inc(2 * n); thread.getBucketDiff.count.inc(4 * n); diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 96618eb9206..4911ad88692 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -1421,6 +1421,14 @@ void FileStorManagerTest::do_test_delete_bucket(bool use_throttled_delete) { StorBucketDatabase::WrappedEntry entry(_node->getStorageBucketDatabase().get(bid, "foo")); EXPECT_FALSE(entry.exists()); } + if (use_throttled_delete) { + auto& metrics = thread_metrics_of(*c.manager)->remove_by_gid; + EXPECT_EQ(metrics.failed.getValue(), 0); + EXPECT_EQ(metrics.count.getValue(), 1); + // We can't reliably test the actual latency here without wiring mock clock bumping into + // the async remove by GID execution, but we can at least test that we updated the metric. + EXPECT_EQ(metrics.latency.getCount(), 1); + } } // TODO remove once throttled behavior is the default diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp index d5d4accd70a..7dea2e621e1 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp @@ -5,6 +5,7 @@ #include <vespa/vdslib/state/cluster_state_bundle.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/vespalib/util/backtrace.h> #include <cassert> #include <vespa/log/log.h> @@ -34,7 +35,10 @@ DistributorBucketSpace & DistributorBucketSpaceRepo::get(BucketSpace bucketSpace) { auto itr = _map.find(bucketSpace); - assert(itr != _map.end()); + if (itr == _map.end()) [[unlikely]] { + LOG(error, "Bucket space %zu does not have a valid mapping. %s", bucketSpace.getId(), vespalib::getStackTrace(0).c_str()); + abort(); + } return *itr->second; } @@ -42,7 +46,10 @@ const DistributorBucketSpace & DistributorBucketSpaceRepo::get(BucketSpace bucketSpace) const { auto itr = _map.find(bucketSpace); - assert(itr != _map.end()); + if (itr == _map.end()) [[unlikely]] { + LOG(error, "Bucket space %zu does not have a valid mapping. %s", bucketSpace.getId(), vespalib::getStackTrace(0).c_str()); + abort(); + } return *itr->second; } diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index bb3952c8c33..8e962a59809 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -300,14 +300,25 @@ AsyncHandler::handle_delete_bucket_throttling(api::DeleteBucketCommand& cmd, Mes }); _spi.deleteBucketAsync(spi_bucket, std::make_unique<ResultTaskOperationDone>(_sequencedExecutor, bucket.getBucketId(), std::move(task))); }); + auto& throttler = _env._fileStorHandler.operation_throttler(); + auto* remove_by_gid_metric = &_env._metrics.remove_by_gid; + for (auto& meta : meta_entries) { auto token = throttler.blocking_acquire_one(); + remove_by_gid_metric->count.inc(); std::vector<spi::DocTypeGidAndTimestamp> to_remove = {{meta->getDocumentType(), meta->getGid(), meta->getTimestamp()}}; - auto task = makeResultTask([bucket = cmd.getBucket(), token = std::move(token), invoke_delete_on_zero_refs]([[maybe_unused]] spi::Result::UP ignored) { - LOG(spam, "%s: completed removeByGidAsync operation", bucket.toString().c_str()); - // Nothing else clever to do here. Throttle token and deleteBucket dispatch refs dropped implicitly. - }); + auto task = makeResultTask([bucket = cmd.getBucket(), token = std::move(token), + invoke_delete_on_zero_refs, remove_by_gid_metric, + op_timer = framework::MilliSecTimer(_env._component.getClock())] + (spi::Result::UP result) { + if (result->hasError()) { + remove_by_gid_metric->failed.inc(); + } + remove_by_gid_metric->latency.addValue(op_timer.getElapsedTimeAsDouble()); + LOG(spam, "%s: completed removeByGidAsync operation", bucket.toString().c_str()); + // Nothing else clever to do here. Throttle token and deleteBucket dispatch refs dropped implicitly. + }); LOG(spam, "%s: about to invoke removeByGidAsync(%s, %s, %zu)", cmd.getBucket().toString().c_str(), vespalib::string(meta->getDocumentType()).c_str(), meta->getGid().toString().c_str(), meta->getTimestamp().getValue()); _spi.removeByGidAsync(spi_bucket, std::move(to_remove), std::make_unique<ResultTaskOperationDone>(_sequencedExecutor, cmd.getBucketId(), std::move(task))); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp index 6072a1dbc3e..9ed15605fb2 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp @@ -154,13 +154,11 @@ FileStorThreadMetrics::FileStorThreadMetrics(const std::string& name, const std: removeLocation("remove_location", "Remove location", this), statBucket("stat_bucket", "Stat bucket", this), update(this), - revert("revert", "Revert", this), createIterator("createiterator", {}, this), visit(this), createBuckets("createbuckets", "Number of buckets that has been created.", this), deleteBuckets("deletebuckets", "Number of buckets that has been deleted.", this), - repairs("bucketverified", "Number of times buckets have been checked.", this), - repairFixed("bucketfixed", {}, "Number of times bucket has been fixed because of corruption", this), + remove_by_gid("remove_by_gid", "Internal single-document remove operations used by DeleteBucket", this), recheckBucketInfo("recheckbucketinfo", "Number of times bucket info has been explicitly " "rechecked due to buckets being marked modified by " @@ -169,11 +167,6 @@ FileStorThreadMetrics::FileStorThreadMetrics(const std::string& name, const std: splitBuckets("splitbuckets", "Number of times buckets have been split.", this), joinBuckets("joinbuckets", "Number of times buckets have been joined.", this), setBucketStates("setbucketstates", "Number of times buckets have been activated or deactivated.", this), - movedBuckets("movedbuckets", "Number of buckets moved between disks", this), - readBucketList("readbucketlist", "Number of read bucket list requests", this), - readBucketInfo("readbucketinfo", "Number of read bucket info requests", this), - internalJoin("internaljoin", "Number of joins to join buckets on multiple disks during " - "storage initialization.", this), mergeBuckets("mergebuckets", "Number of times buckets have been merged.", this), getBucketDiff("getbucketdiff", "Number of getbucketdiff commands that have been processed.", this), applyBucketDiff("applybucketdiff", "Number of applybucketdiff commands that have been processed.", this), @@ -211,18 +204,11 @@ FileStorMetrics::FileStorMetrics() throttle_window_size("throttle_window_size", {}, "Current size of async operation throttler window size", this), throttle_waiting_threads("throttle_waiting_threads", {}, "Number of threads waiting to acquire a throttle token", this), throttle_active_tokens("throttle_active_tokens", {}, "Current number of active throttle tokens", this), - waitingForLockHitRate("waitingforlockrate", {}, - "Amount of times a filestor thread has needed to wait for " - "lock to take next message in queue.", this), active_operations(this), - directoryEvents("directoryevents", {}, "Number of directory events received.", this), - partitionEvents("partitionevents", {}, "Number of partition events received.", this), - diskEvents("diskevents", {}, "Number of disk events received.", this), bucket_db_init_latency("bucket_db_init_latency", {}, "Time taken (in ms) to initialize bucket databases with " "information from the persistence provider", this) { pendingMerges.unsetOnZeroValue(); - waitingForLockHitRate.unsetOnZeroValue(); } FileStorMetrics::~FileStorMetrics() = default; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h index 85a3813c9eb..75182705323 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h @@ -1,11 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * @class storage::FileStorMetrics - * @ingroup filestorage - * - * @brief Metrics for the file store threads. - * - * @version $Id$ + * Metrics for the peristence threads. */ #pragma once @@ -98,21 +93,15 @@ struct FileStorThreadMetrics : public metrics::MetricSet Op removeLocation; Op statBucket; Update update; - OpWithNotFound revert; Op createIterator; Visitor visit; Op createBuckets; Op deleteBuckets; - Op repairs; - metrics::LongCountMetric repairFixed; + Op remove_by_gid; Op recheckBucketInfo; Op splitBuckets; Op joinBuckets; Op setBucketStates; - Op movedBuckets; - Op readBucketList; - Op readBucketInfo; - Op internalJoin; Op mergeBuckets; Op getBucketDiff; Op applyBucketDiff; @@ -148,11 +137,7 @@ struct FileStorMetrics : public metrics::MetricSet metrics::LongAverageMetric throttle_window_size; metrics::LongAverageMetric throttle_waiting_threads; metrics::LongAverageMetric throttle_active_tokens; - metrics::DoubleAverageMetric waitingForLockHitRate; ActiveOperationsMetrics active_operations; - metrics::LongCountMetric directoryEvents; - metrics::LongCountMetric partitionEvents; - metrics::LongCountMetric diskEvents; metrics::LongAverageMetric bucket_db_init_latency; FileStorMetrics(); |