diff options
560 files changed, 8700 insertions, 7319 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 440ecb18024..4fbc756236d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,7 @@ project(vespa CXX C) vespa_use_default_vespa_unprivileged() vespa_use_default_cmake_install_prefix() vespa_use_default_vespa_user() +vespa_use_default_vespa_group() vespa_use_default_build_settings() # allows import of project in CLion on OSX diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/Util.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/Util.java index 2c08936aff0..a5dde14532e 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/Util.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/Util.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.abicheck.collector; +import org.objectweb.asm.Opcodes; + import java.util.ArrayList; -import java.util.Arrays; import java.util.List; -import org.objectweb.asm.Opcodes; public class Util { - public static final List<AccessFlag> classFlags = Arrays.asList( + public static final List<AccessFlag> classFlags = List.of( AccessFlag.make(Opcodes.ACC_PUBLIC, "public"), AccessFlag.make(Opcodes.ACC_PRIVATE, "private"), AccessFlag.make(Opcodes.ACC_PROTECTED, "protected"), @@ -19,12 +19,13 @@ public class Util { AccessFlag.make(Opcodes.ACC_SYNTHETIC, "synthetic"), // FIXME: Do we want this? AccessFlag.make(Opcodes.ACC_ANNOTATION, "annotation"), AccessFlag.make(Opcodes.ACC_ENUM, "enum"), + AccessFlag.make(Opcodes.ACC_RECORD, "record"), // FIXME: Module support // AccessFlag.make(Opcodes.ACC_MODULE, "module") AccessFlag.ignored(Opcodes.ACC_DEPRECATED) ); - public static final List<AccessFlag> methodFlags = Arrays.asList( + public static final List<AccessFlag> methodFlags = List.of( AccessFlag.make(Opcodes.ACC_PUBLIC, "public"), AccessFlag.make(Opcodes.ACC_PRIVATE, "private"), AccessFlag.make(Opcodes.ACC_PROTECTED, "protected"), @@ -40,7 +41,7 @@ public class Util { AccessFlag.ignored(Opcodes.ACC_DEPRECATED) ); - public static final List<AccessFlag> fieldFlags = Arrays.asList( + public static final List<AccessFlag> fieldFlags = List.of( AccessFlag.make(Opcodes.ACC_PUBLIC, "public"), AccessFlag.make(Opcodes.ACC_PRIVATE, "private"), AccessFlag.make(Opcodes.ACC_PROTECTED, "protected"), diff --git a/build_settings.cmake b/build_settings.cmake index e36c56eb9e6..2bf6f93b815 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -178,6 +178,10 @@ else() set(VESPA_USER "vespa") endif() +if(NOT DEFINED VESPA_GROUP) + set(VESPA_GROUP "vespa") +endif() + if(VESPA_UNPRIVILEGED) else() set(VESPA_UNPRIVILEGED "no") diff --git a/client/js/app/.eslintignore b/client/js/app/.eslintignore new file mode 100644 index 00000000000..954e6814384 --- /dev/null +++ b/client/js/app/.eslintignore @@ -0,0 +1,9 @@ +node_modules/* +public/* +storybook-static/* +dist/* +build/* +*.css +*.json +*.png +*.svg diff --git a/client/js/app/.eslintrc.json b/client/js/app/.eslintrc.json new file mode 100644 index 00000000000..991fc93d5ae --- /dev/null +++ b/client/js/app/.eslintrc.json @@ -0,0 +1,51 @@ +{ + "env": { + "browser": true, + "es2021": true, + "node": true, + "commonjs": true, + "es6": true, + "jest": true + }, + "extends": [ + "eslint:recommended", + "plugin:react/recommended", + "plugin:import/recommended" + ], + "parserOptions": { + "ecmaFeatures": { + "jsx": true + }, + "ecmaVersion": 2021, + "sourceType": "module" + }, + "plugins": ["prettier", "react", "react-hooks", "unused-imports"], + "rules": { + "strict": 0, + "react/jsx-uses-react": "error", + "react/prop-types": "off", + "react/display-name": "off", + "react-hooks/rules-of-hooks": "error", + "react-hooks/exhaustive-deps": [ + "warn", + { + "additionalHooks": "use(CustomCompare(Callback|Effect)|Cancelable(Layout)?Effect)" + } + ], + "unused-imports/no-unused-imports": "error", + "import/order": ["error", { "newlines-between": "never" }], + "import/no-unassigned-import": ["error", { "allow": ["**/*.css"] }], + "prettier/prettier": "error" + }, + "settings": { + "react": { + "version": "detect" + }, + "import/resolver": { + "node": { + "extensions": [".js", ".jsx"], + "moduleDirectory": ["node_modules", "src"] + } + } + } +} diff --git a/client/js/app/.husky/pre-commit b/client/js/app/.husky/pre-commit new file mode 100755 index 00000000000..35d6918785a --- /dev/null +++ b/client/js/app/.husky/pre-commit @@ -0,0 +1,4 @@ +#!/bin/sh +. "$(dirname "$0")/_/husky.sh" + +npx pretty-quick --staged diff --git a/client/js/app/.prettierrc b/client/js/app/.prettierrc new file mode 100644 index 00000000000..544138be456 --- /dev/null +++ b/client/js/app/.prettierrc @@ -0,0 +1,3 @@ +{ + "singleQuote": true +} diff --git a/client/js/app/index.html b/client/js/app/index.html index b46ab83364e..d245c706374 100644 --- a/client/js/app/index.html +++ b/client/js/app/index.html @@ -2,12 +2,16 @@ <html lang="en"> <head> <meta charset="UTF-8" /> - <link rel="icon" type="image/svg+xml" href="/src/favicon.svg" /> + <link + rel="icon" + type="image/svg+xml" + href="/src/app/assets/img/favicon.svg" + /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> <title>Vite App</title> </head> <body> <div id="root"></div> - <script type="module" src="/src/main.jsx"></script> + <script type="module" src="/src/app/main.jsx"></script> </body> </html> diff --git a/client/js/app/jsconfig.json b/client/js/app/jsconfig.json new file mode 100644 index 00000000000..290508f30d0 --- /dev/null +++ b/client/js/app/jsconfig.json @@ -0,0 +1,7 @@ +{ + "compilerOptions": { + "baseUrl": "src" + }, + "include": ["src"], + "exclude": ["node_modules", "build", "dist"] +} diff --git a/client/js/app/package.json b/client/js/app/package.json index 1bb03614761..8ea59b0cc35 100644 --- a/client/js/app/package.json +++ b/client/js/app/package.json @@ -5,16 +5,35 @@ "scripts": { "dev": "vite", "build": "vite build", - "preview": "vite preview" + "preview": "vite preview", + "prepare": "cd ../../../ && husky install client/js/app/.husky", + "lint": "eslint '{**/*,*}.{js,jsx}'" }, "dependencies": { - "react": "^18.0.0", - "react-dom": "^18.0.0" + "react": "^18", + "react-dom": "^18" }, "devDependencies": { - "@types/react": "^18.0.0", - "@types/react-dom": "^18.0.0", - "@vitejs/plugin-react": "^1.3.0", - "vite": "^2.9.9" + "@fortawesome/fontawesome-svg-core": "^6", + "@fortawesome/free-regular-svg-icons": "^6", + "@fortawesome/free-solid-svg-icons": "^6", + "@fortawesome/react-fontawesome": "^0", + "@mantine/core": "^4", + "@mantine/hooks": "^4", + "@types/react": "^18", + "@types/react-dom": "^18", + "@vitejs/plugin-react": "^1", + "eslint": "^8", + "eslint-plugin-import": "^2", + "eslint-plugin-prettier": "^4", + "eslint-plugin-react": "^7", + "eslint-plugin-react-hooks": "^4", + "eslint-plugin-react-perf": "^3", + "eslint-plugin-unused-imports": "^2", + "husky": "^7", + "prettier": "2", + "pretty-quick": "^3", + "react-router-dom": "^6", + "vite": "^2" } -}
\ No newline at end of file +} diff --git a/client/js/app/src/App.css b/client/js/app/src/App.css deleted file mode 100644 index 8da3fde63d9..00000000000 --- a/client/js/app/src/App.css +++ /dev/null @@ -1,42 +0,0 @@ -.App { - text-align: center; -} - -.App-logo { - height: 40vmin; - pointer-events: none; -} - -@media (prefers-reduced-motion: no-preference) { - .App-logo { - animation: App-logo-spin infinite 20s linear; - } -} - -.App-header { - background-color: #282c34; - min-height: 100vh; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - font-size: calc(10px + 2vmin); - color: white; -} - -.App-link { - color: #61dafb; -} - -@keyframes App-logo-spin { - from { - transform: rotate(0deg); - } - to { - transform: rotate(360deg); - } -} - -button { - font-size: calc(10px + 2vmin); -} diff --git a/client/js/app/src/App.jsx b/client/js/app/src/App.jsx deleted file mode 100644 index 7d4eb10ff83..00000000000 --- a/client/js/app/src/App.jsx +++ /dev/null @@ -1,45 +0,0 @@ -import { useState } from 'react' -import logo from './logo.svg' -import './App.css' - -function App() { - const [count, setCount] = useState(0) - - return ( - <div className="App"> - <header className="App-header"> - <img src={logo} className="App-logo" alt="logo" /> - <p>Hello Vite + React!</p> - <p> - <button type="button" onClick={() => setCount((count) => count + 1)}> - count is: {count} - </button> - </p> - <p> - Edit <code>App.jsx</code> and save to test HMR updates. - </p> - <p> - <a - className="App-link" - href="https://reactjs.org" - target="_blank" - rel="noopener noreferrer" - > - Learn React - </a> - {' | '} - <a - className="App-link" - href="https://vitejs.dev/guide/features.html" - target="_blank" - rel="noopener noreferrer" - > - Vite Docs - </a> - </p> - </header> - </div> - ) -} - -export default App diff --git a/client/js/app/src/app/app.jsx b/client/js/app/src/app/app.jsx new file mode 100644 index 00000000000..9fa912c89a5 --- /dev/null +++ b/client/js/app/src/app/app.jsx @@ -0,0 +1,24 @@ +import React from 'react'; +import { BrowserRouter, Route } from 'react-router-dom'; +import { Layout } from 'app/components'; +import { Home } from 'app/pages/home/home'; +import { QueryBuilder } from 'app/pages/querybuilder/query-builder'; +import { QueryTracer } from 'app/pages/querytracer/query-tracer'; +import { ThemeProvider } from 'app/libs/theme-provider'; +import { Router } from 'app/libs/router'; + +export function App() { + return ( + <BrowserRouter> + <ThemeProvider> + <Layout> + <Router> + <Route path="/" element={<Home />} /> + <Route path="querybuilder" element={<QueryBuilder />} /> + <Route path="querytracer" element={<QueryTracer />} /> + </Router> + </Layout> + </ThemeProvider> + </BrowserRouter> + ); +} diff --git a/client/js/app/src/app/assets/img/favicon.svg b/client/js/app/src/app/assets/img/favicon.svg new file mode 100644 index 00000000000..f5c574a20c5 --- /dev/null +++ b/client/js/app/src/app/assets/img/favicon.svg @@ -0,0 +1,33 @@ +<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" + viewBox="0 0 150 150"> + <defs> + <style> + .cls-1 { + fill: #ff9d4b; + } + .cls-2 { + fill: url(#linear-gradient); + } + .cls-3 { + fill: #1a7db6; + } + .cls-4 { + fill: url(#linear-gradient-2); + } + </style> + <linearGradient id="linear-gradient" x1="-2012.74" y1="44.83" x2="-2057.81" y2="73.4" + gradientTransform="matrix(-1, 0, 0, 1, -1921.57, 0)" gradientUnits="userSpaceOnUse"> + <stop offset="0.01" stop-color="#c6783e" /> + <stop offset="0.54" stop-color="#ff9750" /> + </linearGradient> + <linearGradient id="linear-gradient-2" x1="60.71" y1="104.56" x2="-15.54" y2="63" gradientUnits="userSpaceOnUse"> + <stop offset="0" stop-color="#005a8e" /> + <stop offset="0.54" stop-color="#1a7db6" /> + </linearGradient> + </defs> + <polygon class="cls-1" points="84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10" /> + <polygon class="cls-2" points="84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02" /> + <polygon class="cls-3" + points="65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99" /> + <polygon class="cls-4" points="65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99" /> +</svg> diff --git a/client/js/app/src/app/assets/img/vespa-icon.svg b/client/js/app/src/app/assets/img/vespa-icon.svg new file mode 100644 index 00000000000..f5c574a20c5 --- /dev/null +++ b/client/js/app/src/app/assets/img/vespa-icon.svg @@ -0,0 +1,33 @@ +<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" + viewBox="0 0 150 150"> + <defs> + <style> + .cls-1 { + fill: #ff9d4b; + } + .cls-2 { + fill: url(#linear-gradient); + } + .cls-3 { + fill: #1a7db6; + } + .cls-4 { + fill: url(#linear-gradient-2); + } + </style> + <linearGradient id="linear-gradient" x1="-2012.74" y1="44.83" x2="-2057.81" y2="73.4" + gradientTransform="matrix(-1, 0, 0, 1, -1921.57, 0)" gradientUnits="userSpaceOnUse"> + <stop offset="0.01" stop-color="#c6783e" /> + <stop offset="0.54" stop-color="#ff9750" /> + </linearGradient> + <linearGradient id="linear-gradient-2" x1="60.71" y1="104.56" x2="-15.54" y2="63" gradientUnits="userSpaceOnUse"> + <stop offset="0" stop-color="#005a8e" /> + <stop offset="0.54" stop-color="#1a7db6" /> + </linearGradient> + </defs> + <polygon class="cls-1" points="84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10" /> + <polygon class="cls-2" points="84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02" /> + <polygon class="cls-3" + points="65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99" /> + <polygon class="cls-4" points="65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99" /> +</svg> diff --git a/client/js/app/src/app/assets/img/vespa-logo.svg b/client/js/app/src/app/assets/img/vespa-logo.svg new file mode 100644 index 00000000000..96f8167d790 --- /dev/null +++ b/client/js/app/src/app/assets/img/vespa-logo.svg @@ -0,0 +1,37 @@ +<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 195.67 57.59"> + <defs> + <style> + .cls-1 { + fill: none; + } + + .cls-2 { + fill: #e9e9e9; + } + </style> + </defs> + <g id="Layer_2"> + <g id="Layer_1-2"> + <path class="cls-1" + d="M179,33.76c0,1.91,1.63,3.11,4.13,3.11,3.19,0,5.74-2,5.74-4.81V30.14l-5.46.38C180.55,30.71,179,31.84,179,33.76Z" /> + <path class="cls-1" d="M105,20.29c-3.14,0-5.45,2.3-5.69,5.63h11.22C110.42,22.51,108.2,20.29,105,20.29Z" /> + <path class="cls-1" + d="M157.11,20.86c-3.52,0-5.86,3.1-5.86,7.75s2.34,7.72,5.86,7.72,5.9-3,5.9-7.75S160.67,20.86,157.11,20.86Z" /> + <polygon class="cls-2" points="31.27 24.98 31.27 0 9.32 14.91 9.32 21.27 22.93 30.86 31.27 24.98" /> + <polygon class="cls-2" points="33.27 23.59 47.9 13.28 47.9 35.28 55.23 40.45 55.23 14.91 33.27 0 33.27 23.59" /> + <path class="cls-2" + d="M93.14,15.59H89.59a4,4,0,0,0-3.87,2.94L81,35.43h-.14l-4.71-16.9a4,4,0,0,0-3.86-2.94H68.59a.43.43,0,0,0-.4.57l8.73,25.15a.43.43,0,0,0,.4.29h7.14a.43.43,0,0,0,.4-.29l8.69-25.15A.43.43,0,0,0,93.14,15.59Z" /> + <path class="cls-2" + d="M134.68,26.6l-4.89-1.06c-2.34-.55-3.47-1.42-3.47-2.72,0-1.68,1.65-2.79,4.13-2.79s4.16,1.16,4.47,3a.42.42,0,0,0,.42.36h5.55a.43.43,0,0,0,.43-.46c-.35-4.7-4.55-7.78-10.78-7.78-6.59,0-10.88,3.21-10.88,8.1,0,3.8,2.52,6.33,7.41,7.44l4.8,1.08c2.36.57,3.26,1.28,3.26,2.62,0,1.71-1.68,2.77-4.42,2.77s-4.45-1.11-4.9-3.1a.43.43,0,0,0-.41-.33h-5.91a.42.42,0,0,0-.42.48c.58,4.74,4.82,7.84,11.45,7.84,6.8,0,11.46-3.33,11.46-8.39C142,29.79,139.9,27.78,134.68,26.6Z" /> + <path class="cls-2" + d="M159.4,15.29A8.63,8.63,0,0,0,151.32,20h-.14V16a.43.43,0,0,0-.43-.43H144.8a.42.42,0,0,0-.42.43V49.68a.42.42,0,0,0,.42.42h2.46a4,4,0,0,0,4-4V37.23h.15c1.41,3,4.32,4.68,8.07,4.68,6.52,0,10.58-5.06,10.58-13.33S166,15.29,159.4,15.29Zm-2.29,21c-3.52,0-5.86-3-5.86-7.72s2.34-7.75,5.86-7.75,5.9,3.05,5.9,7.72S160.67,36.33,157.11,36.33Z" /> + <path class="cls-2" + d="M184.62,15.15c-6.3,0-10.71,3.21-11.15,7.93a.43.43,0,0,0,.43.45h5.5a.43.43,0,0,0,.42-.31c.46-1.72,2.15-2.9,4.51-2.9,2.84,0,4.56,1.37,4.56,3.87v1.33a.42.42,0,0,1-.4.42l-6.09.38c-6.64.4-10.26,3.07-10.26,7.79s3.76,7.8,8.77,7.8c3.33,0,6.54-1.61,8-4.33H189a4,4,0,0,0,4,4h2.22a.43.43,0,0,0,.43-.43V23.77C195.67,18.57,191.3,15.15,184.62,15.15Zm4.27,16.91c0,2.83-2.55,4.81-5.74,4.81-2.5,0-4.13-1.2-4.13-3.11s1.53-3,4.41-3.24l5.46-.38Z" /> + <path class="cls-2" + d="M104.94,15.14c-7.63,0-12.5,5.29-12.5,13.56s4.85,13.35,12.76,13.35c5.89,0,10.37-2.91,11.57-7.36a.81.81,0,0,0-.77-1h-4.83a.89.89,0,0,0-.72.46c-.81,1.74-2.74,2.78-5.13,2.78a6,6,0,0,1-4.4-1.73,6.43,6.43,0,0,1-1.67-4.55v-.43H116.4a.8.8,0,0,0,.8-.8V28.07C117.2,20.15,112.43,15.14,104.94,15.14ZM99.32,25.92c.24-3.33,2.55-5.63,5.69-5.63s5.41,2.22,5.53,5.63Z" /> + <polygon class="cls-2" + points="34.27 25.51 24.2 32.61 24.2 57.59 46.15 42.68 46.15 33.88 46.15 17.14 34.27 25.51" /> + <polygon class="cls-2" points="21.2 32.08 9.32 23.71 0 17.14 0 42.68 21.95 57.59 21.95 32.61 21.2 32.08" /> + </g> + </g> +</svg> diff --git a/client/js/app/src/app/assets/index.js b/client/js/app/src/app/assets/index.js new file mode 100644 index 00000000000..a62869fa71b --- /dev/null +++ b/client/js/app/src/app/assets/index.js @@ -0,0 +1,2 @@ +export { default as VespaLogo } from 'app/assets/img/vespa-logo.svg'; +export { default as VespaIcon } from 'app/assets/img/vespa-icon.svg'; diff --git a/client/js/app/src/app/components/card-link/card-link.jsx b/client/js/app/src/app/components/card-link/card-link.jsx new file mode 100644 index 00000000000..636c8e35e15 --- /dev/null +++ b/client/js/app/src/app/components/card-link/card-link.jsx @@ -0,0 +1,36 @@ +import React from 'react'; +import { Box } from '@mantine/core'; + +export function CardLink({ + sx, + withBorder = true, + borderStyle = 'solid', + minHeight = '89px', + minWidth = 'auto', + ...props +}) { + return ( + <Box + sx={(theme) => ({ + minHeight, + minWidth, + display: 'grid', + placeContent: 'center', + justifyItems: 'center', + rowGap: '8px', + ...theme.fn.hover({ + cursor: 'pointer', + background: theme.cr.getSubtleBackground(), + border: withBorder + ? `1px ${borderStyle} ${theme.cr.getUiElementBorderAndFocus()}` + : 0, + }), + border: withBorder + ? `1px ${borderStyle} ${theme.cr.getSubtleBorderAndSeparator()}` + : 0, + ...sx, + })} + {...props} + /> + ); +} diff --git a/client/js/app/src/app/components/containers/container.jsx b/client/js/app/src/app/components/containers/container.jsx new file mode 100644 index 00000000000..fb30b180a3d --- /dev/null +++ b/client/js/app/src/app/components/containers/container.jsx @@ -0,0 +1,14 @@ +import React from 'react'; +import { Box } from '@mantine/core'; + +export function Container({ sx, ...props }) { + return ( + <Box + sx={() => ({ + display: 'grid', + ...sx, + })} + {...props} + /> + ); +} diff --git a/client/js/app/src/app/components/containers/content.jsx b/client/js/app/src/app/components/containers/content.jsx new file mode 100644 index 00000000000..bbf51a063f2 --- /dev/null +++ b/client/js/app/src/app/components/containers/content.jsx @@ -0,0 +1,32 @@ +import React from 'react'; +import { Paper, Stack, Box } from '@mantine/core'; + +export function Content({ + transparent, + withBorder, + padding, + borderStyle = 'solid', + stack = true, + sx, + ...props +}) { + const Wrapper = stack ? Stack : Box; + return ( + <Paper + sx={(theme) => ({ + background: transparent && 'transparent', + border: withBorder + ? `1px ${borderStyle} ${theme.cr.getSubtleBorderAndSeparator()}` + : 0, + })} + > + <Wrapper + sx={(theme) => ({ + padding: padding ?? theme.spacing.md, + ...sx, + })} + {...props} + /> + </Paper> + ); +} diff --git a/client/js/app/src/app/components/containers/message.jsx b/client/js/app/src/app/components/containers/message.jsx new file mode 100644 index 00000000000..abe3286c021 --- /dev/null +++ b/client/js/app/src/app/components/containers/message.jsx @@ -0,0 +1,16 @@ +import React from 'react'; +import { Box } from '@mantine/core'; + +export function Message({ sx, ...props }) { + return ( + <Box + sx={() => ({ + display: 'grid', + placeContent: 'center', + minHeight: '89px', + ...sx, + })} + {...props} + /> + ); +} diff --git a/client/js/app/src/app/components/containers/section.jsx b/client/js/app/src/app/components/containers/section.jsx new file mode 100644 index 00000000000..d9d43a7aa5a --- /dev/null +++ b/client/js/app/src/app/components/containers/section.jsx @@ -0,0 +1,17 @@ +import React from 'react'; +import { Stack } from '@mantine/core'; + +export function Section({ transparent, sx, ...props }) { + return ( + <Stack + sx={(theme) => ({ + padding: theme.spacing.md, + background: transparent + ? 'transparent' + : theme.cr.getSubtleBackground(), + ...sx, + })} + {...props} + /> + ); +} diff --git a/client/js/app/src/app/components/icon/icon.jsx b/client/js/app/src/app/components/icon/icon.jsx new file mode 100644 index 00000000000..e860d293cce --- /dev/null +++ b/client/js/app/src/app/components/icon/icon.jsx @@ -0,0 +1,13 @@ +import React from 'react'; +import { library } from '@fortawesome/fontawesome-svg-core'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { faArrowsToDot, faChartGantt } from '@fortawesome/free-solid-svg-icons'; + +// TODO: use dynamic import + +library.add(faArrowsToDot, faChartGantt); + +export function Icon({ name, type = 'solid', ...rest }) { + const icon = `fa-${type} fa-${name}`; + return <FontAwesomeIcon icon={icon} {...rest} />; +} diff --git a/client/js/app/src/app/components/index.js b/client/js/app/src/app/components/index.js new file mode 100644 index 00000000000..83ac39bc932 --- /dev/null +++ b/client/js/app/src/app/components/index.js @@ -0,0 +1,9 @@ +export { Error } from 'app/components/layout/error'; +export { Layout } from 'app/components/layout/layout'; +export { Message } from 'app/components/containers/message'; +export { Container } from 'app/components/containers/container'; +export { Content } from 'app/components/containers/content'; +export { Section } from 'app/components/containers/section'; +export { Link } from 'app/components/link/link'; +export { CardLink } from 'app/components/card-link/card-link'; +export { Icon } from 'app/components/icon/icon'; diff --git a/client/js/app/src/app/components/layout/error.jsx b/client/js/app/src/app/components/layout/error.jsx new file mode 100644 index 00000000000..95bc78df413 --- /dev/null +++ b/client/js/app/src/app/components/layout/error.jsx @@ -0,0 +1,25 @@ +import React from 'react'; +import { Center } from '@mantine/core'; + +// TODO: make a better page + +function getMessage(code, location) { + const statusCode = + parseInt(code || new URLSearchParams(location?.search).get('code')) || 404; + + switch (statusCode) { + case 403: + return 'Sorry, you are not authorized to view this page.'; + case 404: + return 'Sorry, the page you were looking for does not exist.'; + case 500: + return 'Oops... something went wrong.'; + default: + return 'Unknown error - really, I have no idea what is going on here.'; + } +} + +export function Error({ code, location }) { + const message = getMessage(code, location); + return <Center sx={{ minHeight: '89px' }}>{message}</Center>; +} diff --git a/client/js/app/src/app/components/layout/header-logo.jsx b/client/js/app/src/app/components/layout/header-logo.jsx new file mode 100644 index 00000000000..09c0fcd1334 --- /dev/null +++ b/client/js/app/src/app/components/layout/header-logo.jsx @@ -0,0 +1,12 @@ +import React from 'react'; +import { Image } from '@mantine/core'; +import { Link } from 'react-router-dom'; +import { VespaLogo } from 'app/assets'; + +export function HeaderLogo() { + return ( + <Link to="/"> + <Image height={34} src={VespaLogo} /> + </Link> + ); +} diff --git a/client/js/app/src/app/components/layout/header.jsx b/client/js/app/src/app/components/layout/header.jsx new file mode 100644 index 00000000000..b4eac293826 --- /dev/null +++ b/client/js/app/src/app/components/layout/header.jsx @@ -0,0 +1,21 @@ +import React from 'react'; +import { Header as MantineHeader } from '@mantine/core'; +import { HeaderLogo } from 'app/components/layout/header-logo'; + +export function Header() { + return ( + <MantineHeader + height={55} + sx={(theme) => ({ + display: 'flex', + alignItems: 'center', + paddingLeft: theme.spacing.md, + paddingRight: theme.spacing.md, + background: theme.cr.getSolidBackground(), + borderBottom: `1px solid ${theme.cr.getSubtleBorderAndSeparator()}`, + })} + > + <HeaderLogo /> + </MantineHeader> + ); +} diff --git a/client/js/app/src/app/components/layout/layout.jsx b/client/js/app/src/app/components/layout/layout.jsx new file mode 100644 index 00000000000..f0f5a0594c0 --- /dev/null +++ b/client/js/app/src/app/components/layout/layout.jsx @@ -0,0 +1,11 @@ +import React from 'react'; +import { AppShell } from '@mantine/core'; +import { Header } from 'app/components/layout/header'; + +export function Layout({ children }) { + return ( + <AppShell header={<Header />} fixed> + {children} + </AppShell> + ); +} diff --git a/client/js/app/src/app/components/link/link.jsx b/client/js/app/src/app/components/link/link.jsx new file mode 100644 index 00000000000..288174d21be --- /dev/null +++ b/client/js/app/src/app/components/link/link.jsx @@ -0,0 +1,23 @@ +import React from 'react'; +import { Link as InternalLink } from 'react-router-dom'; +import { Anchor } from '@mantine/core'; + +export const isInternalLink = (link) => { + if (!link) return false; + return !/^[a-z]+:\/\//.test(link); +}; + +export function Link({ to, api = false, ...props }) { + const internal = !api && isInternalLink(to); + + if (!props.download && to && internal) + return <Anchor component={InternalLink} to={to} {...props} />; + + const fixedProps = Object.assign( + to ? { href: (api ? window.config.api : '') + to } : {}, + to && !internal && { target: '_blank', rel: 'noopener noreferrer' }, + props + ); + + return <Anchor {...fixedProps} />; +} diff --git a/client/js/app/src/app/libs/router.jsx b/client/js/app/src/app/libs/router.jsx new file mode 100644 index 00000000000..c942470d057 --- /dev/null +++ b/client/js/app/src/app/libs/router.jsx @@ -0,0 +1,52 @@ +import React from 'react'; +import { Routes, Route, useParams, Navigate } from 'react-router-dom'; +import { Error } from 'app/components'; + +const mainTitle = 'Vespa App'; + +function TitledRoute({ element, title, default: isDefault, ...props }) { + const params = useParams(); + const clone = React.cloneElement(element, Object.assign(props, params)); + if (title != null) { + const titleStr = typeof title === 'function' ? title(params) : title; + document.title = titleStr.endsWith(mainTitle) + ? titleStr + : `${titleStr} - ${mainTitle}`; + } else if (isDefault) { + // Reset the title if title is not set and this is a default router + document.title = mainTitle; + } + + return clone; +} + +export function Router({ children }) { + // If there is only one route then this comes as an object. + if (!Array.isArray(children)) children = [children]; + children = children.filter(({ props }) => props.enabled ?? true); + + if (!children.some((child) => child.props.default)) + children.push(<Error code={404} default />); + + return ( + <Routes> + {children.map(({ props, ...element }, i) => ( + <Route + key={`${i}-${props.path}`} + path={props.default ? '*' : props.path} + element={ + element.type === Redirect ? ( + Object.assign({ props }, element) + ) : ( + <TitledRoute element={element} {...props} /> + ) + } + /> + ))} + </Routes> + ); +} + +export function Redirect({ to, replace }) { + return <Navigate {...{ to, replace }} />; +} diff --git a/client/js/app/src/app/libs/theme-provider.jsx b/client/js/app/src/app/libs/theme-provider.jsx new file mode 100644 index 00000000000..58496ce15c2 --- /dev/null +++ b/client/js/app/src/app/libs/theme-provider.jsx @@ -0,0 +1,24 @@ +import React from 'react'; +import { Global, MantineProvider } from '@mantine/core'; +import { defaultProps, defaultStyles } from 'app/styles/default'; +import { Colors } from 'app/styles/theme/colors'; +import { styles } from 'app/styles/global'; +import { getTheme } from 'app/styles/theme'; + +function setColorResolver(theme) { + if (!theme.cr) theme.cr = new Colors(theme); + return theme; +} + +export function ThemeProvider({ children }) { + return ( + <MantineProvider + styles={defaultStyles} + defaultProps={defaultProps} + theme={getTheme()} + > + <Global styles={(theme) => styles(setColorResolver(theme))} /> + {children} + </MantineProvider> + ); +} diff --git a/client/js/app/src/main.jsx b/client/js/app/src/app/main.jsx index 9af0bb638e4..96514d419e1 100644 --- a/client/js/app/src/main.jsx +++ b/client/js/app/src/app/main.jsx @@ -1,10 +1,9 @@ -import React from 'react' -import ReactDOM from 'react-dom/client' -import App from './App' -import './index.css' +import React from 'react'; +import ReactDOM from 'react-dom/client'; +import { App } from 'app/app'; ReactDOM.createRoot(document.getElementById('root')).render( <React.StrictMode> <App /> </React.StrictMode> -) +); diff --git a/client/js/app/src/app/pages/home/home.jsx b/client/js/app/src/app/pages/home/home.jsx new file mode 100644 index 00000000000..737b07acf3a --- /dev/null +++ b/client/js/app/src/app/pages/home/home.jsx @@ -0,0 +1,31 @@ +import React from 'react'; +import { Container, SimpleGrid, Space } from '@mantine/core'; +import { Link, CardLink, Icon } from 'app/components'; + +// TODO: move SimpleGrid to components + +export function Home() { + return ( + <Container> + <Space h={55} /> + <SimpleGrid + style={{ gridAutoRows: 'minmax(0, 144px)' }} + breakpoints={[ + { maxWidth: 'sm', cols: 2, spacing: 'sm' }, + { maxWidth: 'xs', cols: 1, spacing: 'sm' }, + ]} + spacing="lg" + cols={2} + > + <CardLink component={Link} to="/querybuilder"> + <Icon name="arrows-to-dot" size="2x" /> + query builder + </CardLink> + <CardLink component={Link} to="/querytracer"> + <Icon name="chart-gantt" size="2x" /> + query tracer + </CardLink> + </SimpleGrid> + </Container> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/query-builder.jsx b/client/js/app/src/app/pages/querybuilder/query-builder.jsx new file mode 100644 index 00000000000..fe158fb5b98 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/query-builder.jsx @@ -0,0 +1,6 @@ +import React from 'react'; +import { Container } from 'app/components'; + +export function QueryBuilder() { + return <Container>query builder</Container>; +} diff --git a/client/js/app/src/app/pages/querytracer/query-tracer.jsx b/client/js/app/src/app/pages/querytracer/query-tracer.jsx new file mode 100644 index 00000000000..94c293d37ed --- /dev/null +++ b/client/js/app/src/app/pages/querytracer/query-tracer.jsx @@ -0,0 +1,6 @@ +import React from 'react'; +import { Container } from 'app/components'; + +export function QueryTracer() { + return <Container>query tracer</Container>; +} diff --git a/client/js/app/src/app/styles/default/default-props.js b/client/js/app/src/app/styles/default/default-props.js new file mode 100644 index 00000000000..c25d9009da9 --- /dev/null +++ b/client/js/app/src/app/styles/default/default-props.js @@ -0,0 +1,9 @@ +export const defaultProps = { + Button: { radius: 2 }, + Badge: { size: 'sm' }, + Group: { spacing: 'xs' }, + Paper: { radius: 0 }, + Tooltip: { withArrow: true }, + Table: { verticalSpacing: 8 }, + ActionIcon: { size: 'sm' }, +}; diff --git a/client/js/app/src/app/styles/default/default-styles.js b/client/js/app/src/app/styles/default/default-styles.js new file mode 100644 index 00000000000..d65e4eecaef --- /dev/null +++ b/client/js/app/src/app/styles/default/default-styles.js @@ -0,0 +1,393 @@ +import { SHADE } from 'app/styles/theme/colors'; + +const inputSizes = Object.freeze({ + xs: 28, + sm: 34, + md: 40, + lg: 48, + xl: 58, +}); + +const buttonSizes = Object.freeze({ + xs: { height: inputSizes.xs, padding: '0 13px' }, + sm: { height: inputSizes.sm, padding: '0 17px' }, + md: { height: inputSizes.md, padding: '0 21px' }, + lg: { height: inputSizes.lg, padding: '0 26px' }, + xl: { height: inputSizes.xl, padding: '0 34px' }, + 'compact-xs': { height: 'initial', padding: '2px 5px' }, + 'compact-sm': { height: 'initial', padding: '3px 8px' }, + 'compact-md': { height: 'initial', padding: '5px 8px' }, + 'compact-lg': { height: 'initial', padding: '5px 13px' }, + 'compact-xl': { height: 'initial', padding: '8px 13px' }, +}); + +export const segmentedControlSizes = Object.freeze({ + xs: '1px 8px', + sm: '1px 13px', + md: '3px 13px', + lg: '5px 21px', + xl: '8px 21px', +}); + +const titleStyles = Object.freeze({ + h1: { + wordBreak: 'break-word', + lineHeight: 'calc(1.5 / var(--space) * var(--vspace))', + }, + h2: { + wordBreak: 'break-word', + fontWeight: 'normal', + lineHeight: '1rem', + }, + h3: { + fontWeight: 'normal', + textTransform: 'uppercase', + letterSpacing: '0.1rem', + lineHeight: '1rem', + }, + h4: { + textTransform: 'capitalize', + lineHeight: 'var(--vspace)', + }, + h5: { + fontWeight: 'lighter', + textTransform: 'uppercase', + letterSpacing: '0.15rem', + lineHeight: 'var(--vspace)', + }, + h6: { + fontWeight: 'normal', + fontStyle: 'italic', + letterSpacing: '0 !important', + lineHeight: 'var(--vspace)', + }, +}); + +const getButtonStyles = ({ compact, size }) => { + if (!compact) return buttonSizes[size]; + return buttonSizes[`compact-${size}`]; +}; + +const getTitleStyles = ({ element = 'h1' }) => titleStyles[element]; + +export function getVariantStyles({ fn, white }, color, variant) { + if (variant === 'hover' || variant === 'transparent') { + return { + color: fn.themeColor(color, SHADE.SOLID_BACKGROUND), + backgroundColor: 'transparent', + border: '1px solid transparent', + ...fn.hover( + variant === 'hover' + ? { + backgroundColor: fn.themeColor( + color, + SHADE.UI_ELEMENT_BACKGROUND + ), + } + : {} + ), + }; + } + if (variant === 'light') { + return { + color: fn.themeColor(color, SHADE.LOW_CONTRAST_TEXT), + backgroundColor: fn.themeColor(color, SHADE.HOVERED_ELEMENT_BACKGROUND), + border: '1px solid transparent', + '&:hover': { + color: fn.themeColor(color, SHADE.LOW_CONTRAST_TEXT), + backgroundColor: fn.themeColor( + color, + SHADE.SUBTLE_BORDER_AND_SEPARATOR + ), + }, + }; + } + if (variant === 'outline') { + return { + color: fn.themeColor(color, SHADE.LOW_CONTRAST_TEXT), + backgroundColor: fn.themeColor(color, SHADE.APP_BACKGROUND), + border: `1px solid ${fn.themeColor(color, SHADE.SOLID_BACKGROUND)}`, + '&:hover': { + color: fn.themeColor(color, SHADE.LOW_CONTRAST_TEXT), + borderColor: fn.themeColor(color, SHADE.HOVERED_SOLID_BACKGROUND), + }, + }; + } + if (variant === 'dot') { + return { + color: fn.themeColor('gray', SHADE.LOW_CONTRAST_TEXT), + backgroundColor: 'transparent', + border: `1px solid ${fn.themeColor( + 'gray', + SHADE.UI_ELEMENT_BORDER_AND_FOCUS + )}`, + '&:hover': { + color: fn.themeColor('gray', SHADE.LOW_CONTRAST_TEXT), + borderColor: fn.themeColor('gray', SHADE.UI_ELEMENT_BORDER_AND_FOCUS), + }, + '&::before': { + backgroundColor: fn.themeColor(color, SHADE.SOLID_BACKGROUND), + }, + }; + } + if (variant === 'subtle') { + return { + color: fn.themeColor(color, SHADE.LOW_CONTRAST_TEXT), + backgroundColor: 'transparent', + border: 'transparent', + '&:hover': { + color: fn.themeColor(color, SHADE.LOW_CONTRAST_TEXT), + backgroundColor: fn.themeColor(color, SHADE.HOVERED_ELEMENT_BACKGROUND), + }, + }; + } + if (variant === 'default') { + return { + color: fn.themeColor('gray', SHADE.HIGH_CONTRAST_TEXT), + backgroundColor: fn.themeColor('gray', SHADE.SUBTLE_BACKGROUND), + border: `1px solid ${fn.themeColor( + 'gray', + SHADE.UI_ELEMENT_BORDER_AND_FOCUS + )}`, + '&:hover': { + color: fn.themeColor('gray', SHADE.HIGH_CONTRAST_TEXT), + backgroundColor: fn.themeColor( + 'gray', + SHADE.HOVERED_ELEMENT_BACKGROUND + ), + }, + }; + } + if (variant === 'basic') { + return { + color: fn.themeColor('gray', SHADE.HIGH_CONTRAST_TEXT), + backgroundColor: 'transparent', + border: '1px solid transparent', + '&:hover': { + color: fn.themeColor('gray', SHADE.HIGH_CONTRAST_TEXT), + backgroundColor: 'transparent', + }, + }; + } + return { + color: white, + backgroundColor: fn.themeColor(color, SHADE.SOLID_BACKGROUND), + border: '1px solid transparent', + '&:hover': { + color: white, + backgroundColor: fn.themeColor(color, SHADE.HOVERED_SOLID_BACKGROUND), + }, + }; +} + +function getInputSizes({ fn, size }) { + return { + minHeight: fn.size({ size, sizes: inputSizes }), + height: fn.size({ size, sizes: inputSizes }), + lineHeight: `${fn.size({ size, sizes: inputSizes }) - 2}px`, + }; +} + +function getInputVariantStyles({ fn, cr }, variant, size) { + if (variant === 'unstyled') { + return { + color: cr.getHighContrastText(), + }; + } + if (variant === 'filled') { + return { + backgroundColor: cr.getUiElementBackground(), + '&:focus, &:focus-within': { + borderColor: `${cr.getUiElementBorderAndFocus()} !important`, + }, + ...getInputSizes({ fn, size }), + }; + } + return { + border: `1px solid ${cr.getSubtleBorderAndSeparator()}`, + backgroundColor: cr.getSubtleBackground(), + '&:focus, &:focus-within': { + borderColor: cr.getUiElementBorderAndFocus(), + }, + ...getInputSizes({ fn, size }), + }; +} + +export const defaultStyles = { + AppShell: () => ({ + main: { + maxWidth: '1920px', + margin: '0 auto', + }, + }), + ActionIcon: (theme, { color }) => ({ + root: { + '&:disabled': { + color: theme.fn.themeColor('gray', SHADE.SOLID_BACKGROUND), + backgroundColor: 'transparent', + borderColor: 'transparent', + opacity: 0.55, + }, + }, + light: getVariantStyles(theme, color, 'light'), + filled: getVariantStyles(theme, color, 'filled'), + outline: getVariantStyles(theme, color, 'outline'), + default: getVariantStyles(theme, color, 'default'), + hover: getVariantStyles(theme, color, 'hover'), + transparent: getVariantStyles(theme, color, 'transparent'), + }), + Badge: (theme, { color }) => ({ + light: getVariantStyles(theme, color, 'light'), + filled: getVariantStyles(theme, color, 'filled'), + outline: getVariantStyles(theme, color, 'outline'), + dot: getVariantStyles(theme, color, 'dot'), + }), + Button: (theme, { color, compact, size }) => ({ + root: getButtonStyles({ compact, size }), + light: getVariantStyles(theme, color, 'light'), + filled: getVariantStyles(theme, color, 'filled'), + outline: getVariantStyles(theme, color, 'outline'), + subtle: getVariantStyles(theme, color, 'subtle'), + default: getVariantStyles(theme, color, 'default'), + leftIcon: { marginRight: 3 }, + rightIcon: { marginLeft: 3 }, + }), + Divider: (theme) => ({ + horizontal: { + borderTopColor: theme.cr.getSubtleBorderAndSeparator(), + }, + }), + Input: (theme, { invalid, size }) => ({ + defaultVariant: { ...getInputVariantStyles(theme, 'default', size) }, + filledVariant: { ...getInputVariantStyles(theme, 'filled', size) }, + unstyledVariant: { ...getInputVariantStyles(theme, 'unstyled', size) }, + invalid: { + color: theme.fn.themeColor('red', SHADE.SOLID_BACKGROUND), + borderColor: theme.fn.themeColor('red', SHADE.SOLID_BACKGROUND), + '&::placeholder': { + color: theme.fn.themeColor('red', SHADE.SOLID_BACKGROUND), + }, + }, + disabled: { + backgroundColor: theme.fn.themeColor('gray', SHADE.SOLID_BACKGROUND), + color: theme.fn.themeColor('gray', SHADE.LOW_CONTRAST_TEXT), + '&::placeholder': { + color: theme.fn.themeColor('gray', SHADE.LOW_CONTRAST_TEXT), + }, + }, + icon: { + color: invalid + ? theme.fn.themeColor('red', SHADE.SOLID_BACKGROUND) + : theme.cr.getUiElementBorderAndFocus(), + }, + }), + Paper: (theme) => ({ + root: { + color: theme.cr.getHighContrastText(), + backgroundColor: theme.cr.getUiElementBackground(), + }, + }), + Popover: (theme) => ({ + arrow: { + borderColor: theme.cr.getUiElementBorderAndFocus(), + background: theme.cr.getSubtleBackground(), + }, + popover: { + background: theme.cr.getSubtleBackground(), + }, + body: { + border: `1px solid ${theme.cr.getUiElementBorderAndFocus()}`, + whiteSpace: 'nowrap', + }, + header: { + borderBottom: `1px solid ${theme.cr.getUiElementBorderAndFocus()}`, + }, + }), + Progress: (theme) => ({ + root: { + display: 'flex', + backgroundColor: theme.fn.themeColor( + 'gray', + SHADE.SUBTLE_BORDER_AND_SEPARATOR + ), + }, + bar: { + position: 'unset', + minWidth: '2em', + }, + }), + SegmentedControl: (theme, { size }) => ({ + root: { + backgroundColor: theme.cr.getUiElementBackground(), + }, + label: { + color: theme.cr.getLowContrastText(), + padding: + segmentedControlSizes[size in segmentedControlSizes ? size : 'sm'], + }, + control: { + '&:not(:first-of-type)': { + borderColor: theme.cr.getUiElementBorderAndFocus(), + }, + }, + }), + Select: (theme, { variant, size }) => ({ + input: { ...getInputVariantStyles(theme, variant, size) }, + dropdown: { + backgroundColor: theme.cr.getSubtleBackground(), + border: `1px solid ${theme.cr.getUiElementBorderAndFocus()}`, + borderRadius: theme.radius.xs, + }, + item: { + color: theme.cr.getHighContrastText(), + borderRadius: 0, + }, + selected: { + backgroundColor: theme.cr.getUiElementBackground(), + color: theme.cr.getHighContrastText(), + }, + hovered: { + backgroundColor: theme.cr.getHoveredUiElementBackground(), + }, + }), + Table: (theme) => ({ + root: { + color: theme.cr.getHighContrastText(), + '& thead tr th': { + textTransform: 'uppercase', + fontSize: theme.fontSizes.xs, + color: theme.cr.getLowContrastText(), + borderBottom: `1px solid ${theme.cr.getSubtleBorderAndSeparator()}`, + }, + '& tbody tr td': { + borderBottom: `1px solid ${theme.cr.getSubtleBorderAndSeparator()}`, + }, + '& thead tr:hover': { + backgroundColor: 'transparent !important', + }, + }, + }), + Text: (theme, { color, variant }) => ({ + root: { + color: theme.cr.getText(color, variant), + ...theme.fn.hover({ + color: theme.cr.getText(color, variant), + textDecorationColor: theme.fn.themeColor( + color, + SHADE.UI_ELEMENT_BORDER_AND_FOCUS + ), + }), + }, + }), + Title: (theme, { element }) => ({ + root: { + '--space': theme.lineHeight, + '--vspace': 'calc(var(--space) * 1rem)', + ...getTitleStyles({ element }), + }, + }), + Tooltip: (theme) => ({ + body: { ...theme.cr.getTooltip() }, + arrow: { ...theme.cr.getTooltip() }, + }), +}; diff --git a/client/js/app/src/app/styles/default/index.js b/client/js/app/src/app/styles/default/index.js new file mode 100644 index 00000000000..5ffc102d3d4 --- /dev/null +++ b/client/js/app/src/app/styles/default/index.js @@ -0,0 +1,2 @@ +export { defaultProps } from 'app/styles/default/default-props'; +export { defaultStyles } from 'app/styles/default/default-styles'; diff --git a/client/js/app/src/app/styles/global.js b/client/js/app/src/app/styles/global.js new file mode 100644 index 00000000000..900ae5d9304 --- /dev/null +++ b/client/js/app/src/app/styles/global.js @@ -0,0 +1,39 @@ +export const styles = (theme) => ({ + '*, *::before, *::after': { + boxSizing: 'border-box', + }, + + '*': { + margin: '0', + }, + + html: { + height: '100%', + }, + + body: { + height: '100%', + WebkitFontSmoothing: 'antialiased', + lineHeight: theme.lineHeight, + background: theme.cr.getAppBackground(), + color: theme.cr.getHighContrastText(), + ...theme.fn.fontStyles(), + }, + + 'img, picture, video, canvas, svg': { + display: 'block', + }, + + 'input, button, textarea, select': { + font: 'inherit', + }, + + 'p, h1, h2, h3, h4, h5, h6': { + overflowWrap: 'break-word', + }, + + '#root': { + height: '100%', + isolation: 'isolate', + }, +}); diff --git a/client/js/app/src/app/styles/theme/colors.js b/client/js/app/src/app/styles/theme/colors.js new file mode 100644 index 00000000000..5736b514329 --- /dev/null +++ b/client/js/app/src/app/styles/theme/colors.js @@ -0,0 +1,69 @@ +export const SHADE = Object.freeze({ + APP_BACKGROUND: 0, + SUBTLE_BACKGROUND: 1, + UI_ELEMENT_BACKGROUND: 2, + HOVERED_ELEMENT_BACKGROUND: 3, + SUBTLE_BORDER_AND_SEPARATOR: 4, + UI_ELEMENT_BORDER_AND_FOCUS: 5, + SOLID_BACKGROUND: 6, + HOVERED_SOLID_BACKGROUND: 7, + LOW_CONTRAST_TEXT: 8, + HIGH_CONTRAST_TEXT: 9, +}); + +export class Colors { + constructor(theme) { + this.theme = theme; + this.themeColor = theme.colors['blue']; + } + + getAppBackground() { + return this.themeColor[SHADE.APP_BACKGROUND]; + } + + getSubtleBackground() { + return this.themeColor[SHADE.SUBTLE_BACKGROUND]; + } + + getUiElementBackground() { + return this.themeColor[SHADE.UI_ELEMENT_BACKGROUND]; + } + + getHoveredUiElementBackground() { + return this.themeColor[SHADE.HOVERED_ELEMENT_BACKGROUND]; + } + + getSubtleBorderAndSeparator() { + return this.themeColor[SHADE.SUBTLE_BORDER_AND_SEPARATOR]; + } + + getUiElementBorderAndFocus() { + return this.themeColor[SHADE.UI_ELEMENT_BORDER_AND_FOCUS]; + } + + getSolidBackground(color) { + return this.theme.fn.themeColor(color, SHADE.SOLID_BACKGROUND); + } + + getHoveredSolidBackground(color) { + return this.theme.fn.themeColor(color, SHADE.HOVERED_SOLID_BACKGROUND); + } + + getLowContrastText() { + return this.theme.colors.gray[SHADE.LOW_CONTRAST_TEXT]; + } + + getHighContrastText() { + return this.theme.colors.gray[SHADE.HIGH_CONTRAST_TEXT]; + } + + getText(color, variant) { + return color === 'dimmed' + ? this.theme.fn.themeColor('gray', SHADE.SOLID_BACKGROUND) + : color in this.theme.colors + ? this.theme.fn.themeColor(color, SHADE.LOW_CONTRAST_TEXT) + : variant === 'link' + ? this.theme.fn.themeColor(color, SHADE.SOLID_BACKGROUND) + : color || 'inherit'; + } +} diff --git a/client/js/app/src/app/styles/theme/common-colors.js b/client/js/app/src/app/styles/theme/common-colors.js new file mode 100644 index 00000000000..fa942d3abff --- /dev/null +++ b/client/js/app/src/app/styles/theme/common-colors.js @@ -0,0 +1,158 @@ +export const commonColors = { + gray: [ + '#fcfcfc', + '#f8f8f8', + '#f3f3f3', + '#ededed', + '#e8e8e8', + '#e2e2e2', + '#8f8f8f', + '#858585', + '#6f6f6f', + '#171717', + ], + red: [ + '#fffcfc', + '#fff8f8', + '#ffefef', + '#ffe5e5', + '#fdd8d8', + '#f9c6c6', + '#e5484d', + '#dc3d43', + '#cd2b31', + '#381316', + ], + pink: [ + '#fffcfe', + '#fff7fc', + '#feeef8', + '#fce5f3', + '#f9d8ec', + '#f3c6e2', + '#d6409f', + '#d23197', + '#cd1d8d', + '#3b0a2a', + ], + grape: [ + '#fefcfe', + '#fdfaff', + '#f9f1fe', + '#f3e7fc', + '#eddbf9', + '#e3ccf4', + '#8e4ec6', + '#8445bc', + '#793aaf', + '#2b0e44', + ], + violet: [ + '#fdfcfe', + '#fbfaff', + '#f5f2ff', + '#ede9fe', + '#e4defc', + '#d7cff9', + '#6e56cf', + '#644fc1', + '#5746af', + '#20134b', + ], + indigo: [ + '#fdfdfe', + '#f8faff', + '#f0f4ff', + '#e6edfe', + '#d9e2fc', + '#c6d4f9', + '#3e63dd', + '#3a5ccc', + '#3451b2', + '#101d46', + ], + blue: [ + '#f5fbff', + '#feffff', + '#edf8ff', + '#e1f4ff', + '#ceecfe', + '#b7e0f8', + '#00598c', + '#00507e', + '#00436a', + '#002033', + ], + cyan: [ + '#fafdfe', + '#f2fcfd', + '#e7f9fb', + '#d8f3f6', + '#c4eaef', + '#aadee6', + '#05a2c2', + '#0894b3', + '#0c7792', + '#04313c', + ], + teal: [ + '#fafefd', + '#f1fcfa', + '#e7f9f5', + '#d9f3ee', + '#c7ebe5', + '#afdfd7', + '#12a594', + '#0e9888', + '#067a6f', + '#10302b', + ], + green: [ + '#fbfefc', + '#f2fcf5', + '#e9f9ee', + '#ddf3e4', + '#ccebd7', + '#b4dfc4', + '#30a46c', + '#299764', + '#18794e', + '#153226', + ], + lime: [ + '#fcfdfa', + '#f7fcf0', + '#eefadc', + '#e4f7c7', + '#d7f2b0', + '#c9e894', + '#99d52a', + '#93c926', + '#5d770d', + '#263209', + ], + yellow: [ + '#fdfdf9', + '#fffce8', + '#fffbd1', + '#fff8bb', + '#fef2a4', + '#f9e68c', + '#f5d90a', + '#f7ce00', + '#946800', + '#35290f', + ], + orange: [ + '#fefcfb', + '#fef8f4', + '#fff1e7', + '#ffe8d7', + '#ffdcc3', + '#ffcca7', + '#f76808', + '#ed5f00', + '#bd4b00', + '#451e11', + ], +}; diff --git a/client/js/app/src/app/styles/theme/common.js b/client/js/app/src/app/styles/theme/common.js new file mode 100644 index 00000000000..93a08721e5c --- /dev/null +++ b/client/js/app/src/app/styles/theme/common.js @@ -0,0 +1,35 @@ +export const common = { + primaryShade: 6, + loader: 'oval', + white: '#fff', + black: '#303030', + defaultRadius: 'xs', + primaryColor: 'blue', + lineHeight: 1.5, + fontFamily: 'Lato, sans-serif', + shadows: { + xs: '0 1px 3px rgba(0, 0, 0, 0.05), 0 1px 2px rgba(0, 0, 0, 0.1)', + sm: '0 1px 3px rgba(0, 0, 0, 0.05), rgba(0, 0, 0, 0.05) 0px 10px 15px -5px, rgba(0, 0, 0, 0.04) 0px 7px 7px -5px', + md: '0 1px 3px rgba(0, 0, 0, 0.05), rgba(0, 0, 0, 0.05) 0px 20px 25px -5px, rgba(0, 0, 0, 0.04) 0px 10px 10px -5px', + lg: '0 1px 3px rgba(0, 0, 0, 0.05), rgba(0, 0, 0, 0.05) 0px 28px 23px -7px, rgba(0, 0, 0, 0.04) 0px 12px 12px -7px', + xl: '0 1px 3px rgba(0, 0, 0, 0.05), rgba(0, 0, 0, 0.05) 0px 36px 28px -7px, rgba(0, 0, 0, 0.04) 0px 17px 17px -7px', + }, + fontSizes: { xs: 12, sm: 14, md: 16, lg: 18, xl: 20 }, + radius: { xs: 5, sm: 8, md: 13, lg: 21, xl: 34 }, + spacing: { xs: 5, sm: 8, md: 13, lg: 21, xl: 34 }, + breakpoints: { xs: 576, sm: 768, md: 992, lg: 1200, xl: 1400 }, + headings: { + fontFamily: 'Lato, sans-serif', + sizes: { + h1: { fontSize: '1.5rem' }, + h2: { fontSize: '1.3333rem' }, + h3: { fontSize: '1.125rem' }, + h4: { fontSize: '1rem' }, + h5: { fontSize: '0.9375rem' }, + h6: { fontSize: '0.875rem' }, + }, + }, + other: {}, + datesLocale: 'en', + fn: {}, +}; diff --git a/client/js/app/src/app/styles/theme/index.js b/client/js/app/src/app/styles/theme/index.js new file mode 100644 index 00000000000..fbd74bf13fc --- /dev/null +++ b/client/js/app/src/app/styles/theme/index.js @@ -0,0 +1,6 @@ +import { common } from 'app/styles/theme/common'; +import { commonColors } from 'app/styles/theme/common-colors'; + +export const getTheme = () => { + return { ...common, colors: commonColors }; +}; diff --git a/client/js/app/src/favicon.svg b/client/js/app/src/favicon.svg deleted file mode 100644 index de4aeddc12b..00000000000 --- a/client/js/app/src/favicon.svg +++ /dev/null @@ -1,15 +0,0 @@ -<svg width="410" height="404" viewBox="0 0 410 404" fill="none" xmlns="http://www.w3.org/2000/svg"> -<path d="M399.641 59.5246L215.643 388.545C211.844 395.338 202.084 395.378 198.228 388.618L10.5817 59.5563C6.38087 52.1896 12.6802 43.2665 21.0281 44.7586L205.223 77.6824C206.398 77.8924 207.601 77.8904 208.776 77.6763L389.119 44.8058C397.439 43.2894 403.768 52.1434 399.641 59.5246Z" fill="url(#paint0_linear)"/> -<path d="M292.965 1.5744L156.801 28.2552C154.563 28.6937 152.906 30.5903 152.771 32.8664L144.395 174.33C144.198 177.662 147.258 180.248 150.51 179.498L188.42 170.749C191.967 169.931 195.172 173.055 194.443 176.622L183.18 231.775C182.422 235.487 185.907 238.661 189.532 237.56L212.947 230.446C216.577 229.344 220.065 232.527 219.297 236.242L201.398 322.875C200.278 328.294 207.486 331.249 210.492 326.603L212.5 323.5L323.454 102.072C325.312 98.3645 322.108 94.137 318.036 94.9228L279.014 102.454C275.347 103.161 272.227 99.746 273.262 96.1583L298.731 7.86689C299.767 4.27314 296.636 0.855181 292.965 1.5744Z" fill="url(#paint1_linear)"/> -<defs> -<linearGradient id="paint0_linear" x1="6.00017" y1="32.9999" x2="235" y2="344" gradientUnits="userSpaceOnUse"> -<stop stop-color="#41D1FF"/> -<stop offset="1" stop-color="#BD34FE"/> -</linearGradient> -<linearGradient id="paint1_linear" x1="194.651" y1="8.81818" x2="236.076" y2="292.989" gradientUnits="userSpaceOnUse"> -<stop stop-color="#FFEA83"/> -<stop offset="0.0833333" stop-color="#FFDD35"/> -<stop offset="1" stop-color="#FFA800"/> -</linearGradient> -</defs> -</svg> diff --git a/client/js/app/src/index.css b/client/js/app/src/index.css deleted file mode 100644 index ec2585e8c0b..00000000000 --- a/client/js/app/src/index.css +++ /dev/null @@ -1,13 +0,0 @@ -body { - margin: 0; - font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', - 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', - sans-serif; - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; -} - -code { - font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', - monospace; -} diff --git a/client/js/app/src/logo.svg b/client/js/app/src/logo.svg deleted file mode 100644 index 6b60c1042f5..00000000000 --- a/client/js/app/src/logo.svg +++ /dev/null @@ -1,7 +0,0 @@ -<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 841.9 595.3"> - <g fill="#61DAFB"> - <path d="M666.3 296.5c0-32.5-40.7-63.3-103.1-82.4 14.4-63.6 8-114.2-20.2-130.4-6.5-3.8-14.1-5.6-22.4-5.6v22.3c4.6 0 8.3.9 11.4 2.6 13.6 7.8 19.5 37.5 14.9 75.7-1.1 9.4-2.9 19.3-5.1 29.4-19.6-4.8-41-8.5-63.5-10.9-13.5-18.5-27.5-35.3-41.6-50 32.6-30.3 63.2-46.9 84-46.9V78c-27.5 0-63.5 19.6-99.9 53.6-36.4-33.8-72.4-53.2-99.9-53.2v22.3c20.7 0 51.4 16.5 84 46.6-14 14.7-28 31.4-41.3 49.9-22.6 2.4-44 6.1-63.6 11-2.3-10-4-19.7-5.2-29-4.7-38.2 1.1-67.9 14.6-75.8 3-1.8 6.9-2.6 11.5-2.6V78.5c-8.4 0-16 1.8-22.6 5.6-28.1 16.2-34.4 66.7-19.9 130.1-62.2 19.2-102.7 49.9-102.7 82.3 0 32.5 40.7 63.3 103.1 82.4-14.4 63.6-8 114.2 20.2 130.4 6.5 3.8 14.1 5.6 22.5 5.6 27.5 0 63.5-19.6 99.9-53.6 36.4 33.8 72.4 53.2 99.9 53.2 8.4 0 16-1.8 22.6-5.6 28.1-16.2 34.4-66.7 19.9-130.1 62-19.1 102.5-49.9 102.5-82.3zm-130.2-66.7c-3.7 12.9-8.3 26.2-13.5 39.5-4.1-8-8.4-16-13.1-24-4.6-8-9.5-15.8-14.4-23.4 14.2 2.1 27.9 4.7 41 7.9zm-45.8 106.5c-7.8 13.5-15.8 26.3-24.1 38.2-14.9 1.3-30 2-45.2 2-15.1 0-30.2-.7-45-1.9-8.3-11.9-16.4-24.6-24.2-38-7.6-13.1-14.5-26.4-20.8-39.8 6.2-13.4 13.2-26.8 20.7-39.9 7.8-13.5 15.8-26.3 24.1-38.2 14.9-1.3 30-2 45.2-2 15.1 0 30.2.7 45 1.9 8.3 11.9 16.4 24.6 24.2 38 7.6 13.1 14.5 26.4 20.8 39.8-6.3 13.4-13.2 26.8-20.7 39.9zm32.3-13c5.4 13.4 10 26.8 13.8 39.8-13.1 3.2-26.9 5.9-41.2 8 4.9-7.7 9.8-15.6 14.4-23.7 4.6-8 8.9-16.1 13-24.1zM421.2 430c-9.3-9.6-18.6-20.3-27.8-32 9 .4 18.2.7 27.5.7 9.4 0 18.7-.2 27.8-.7-9 11.7-18.3 22.4-27.5 32zm-74.4-58.9c-14.2-2.1-27.9-4.7-41-7.9 3.7-12.9 8.3-26.2 13.5-39.5 4.1 8 8.4 16 13.1 24 4.7 8 9.5 15.8 14.4 23.4zM420.7 163c9.3 9.6 18.6 20.3 27.8 32-9-.4-18.2-.7-27.5-.7-9.4 0-18.7.2-27.8.7 9-11.7 18.3-22.4 27.5-32zm-74 58.9c-4.9 7.7-9.8 15.6-14.4 23.7-4.6 8-8.9 16-13 24-5.4-13.4-10-26.8-13.8-39.8 13.1-3.1 26.9-5.8 41.2-7.9zm-90.5 125.2c-35.4-15.1-58.3-34.9-58.3-50.6 0-15.7 22.9-35.6 58.3-50.6 8.6-3.7 18-7 27.7-10.1 5.7 19.6 13.2 40 22.5 60.9-9.2 20.8-16.6 41.1-22.2 60.6-9.9-3.1-19.3-6.5-28-10.2zM310 490c-13.6-7.8-19.5-37.5-14.9-75.7 1.1-9.4 2.9-19.3 5.1-29.4 19.6 4.8 41 8.5 63.5 10.9 13.5 18.5 27.5 35.3 41.6 50-32.6 30.3-63.2 46.9-84 46.9-4.5-.1-8.3-1-11.3-2.7zm237.2-76.2c4.7 38.2-1.1 67.9-14.6 75.8-3 1.8-6.9 2.6-11.5 2.6-20.7 0-51.4-16.5-84-46.6 14-14.7 28-31.4 41.3-49.9 22.6-2.4 44-6.1 63.6-11 2.3 10.1 4.1 19.8 5.2 29.1zm38.5-66.7c-8.6 3.7-18 7-27.7 10.1-5.7-19.6-13.2-40-22.5-60.9 9.2-20.8 16.6-41.1 22.2-60.6 9.9 3.1 19.3 6.5 28.1 10.2 35.4 15.1 58.3 34.9 58.3 50.6-.1 15.7-23 35.6-58.4 50.6zM320.8 78.4z"/> - <circle cx="420.9" cy="296.5" r="45.7"/> - <path d="M520.5 78.1z"/> - </g> -</svg> diff --git a/client/js/app/vite.config.js b/client/js/app/vite.config.js index b1b5f91e5ff..de091b18d41 100644 --- a/client/js/app/vite.config.js +++ b/client/js/app/vite.config.js @@ -1,7 +1,23 @@ -import { defineConfig } from 'vite' -import react from '@vitejs/plugin-react' +import * as path from 'path'; +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; // https://vitejs.dev/config/ export default defineConfig({ - plugins: [react()] -}) + build: { + sourcemap: 'hidden', + rollupOptions: { + output: { + manualChunks(id) { + return id.includes('node_modules') ? 'vendor' : 'main'; + }, + }, + }, + }, + plugins: [react()], + resolve: { + alias: { + app: path.resolve(__dirname, 'src/app'), + }, + }, +}); diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 00c958b411e..f1cee14e50e 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -201,6 +201,13 @@ "@babel/plugin-syntax-jsx" "^7.17.12" "@babel/types" "^7.17.12" +"@babel/runtime@^7.10.2", "@babel/runtime@^7.13.10", "@babel/runtime@^7.7.6": + version "7.18.3" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.18.3.tgz#c7b654b57f6f63cf7f8b418ac9ca04408c4579f4" + integrity sha512-38Y8f7YUhce/K7RMwTp7m0uCumpv9hZkitCbBClqQIow1qSbCvGkcegKOXpEWCQLfWmevgRiWokZ1GkpfhbZug== + dependencies: + regenerator-runtime "^0.13.4" + "@babel/template@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.16.7.tgz#8d126c8701fde4d66b264b3eba3d96f07666d155" @@ -234,6 +241,160 @@ "@babel/helper-validator-identifier" "^7.16.7" to-fast-properties "^2.0.0" +"@emotion/cache@11.7.1": + version "11.7.1" + resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.7.1.tgz#08d080e396a42e0037848214e8aa7bf879065539" + integrity sha512-r65Zy4Iljb8oyjtLeCuBH8Qjiy107dOYC6SJq7g7GV5UCQWMObY4SJDPGFjiiVpPrOJ2hmJOoBiYTC7hwx9E2A== + dependencies: + "@emotion/memoize" "^0.7.4" + "@emotion/sheet" "^1.1.0" + "@emotion/utils" "^1.0.0" + "@emotion/weak-memoize" "^0.2.5" + stylis "4.0.13" + +"@emotion/cache@^11.7.1": + version "11.9.3" + resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.9.3.tgz#96638449f6929fd18062cfe04d79b29b44c0d6cb" + integrity sha512-0dgkI/JKlCXa+lEXviaMtGBL0ynpx4osh7rjOXE71q9bIF8G+XhJgvi+wDu0B0IdCVx37BffiwXlN9I3UuzFvg== + dependencies: + "@emotion/memoize" "^0.7.4" + "@emotion/sheet" "^1.1.1" + "@emotion/utils" "^1.0.0" + "@emotion/weak-memoize" "^0.2.5" + stylis "4.0.13" + +"@emotion/hash@^0.8.0": + version "0.8.0" + resolved "https://registry.yarnpkg.com/@emotion/hash/-/hash-0.8.0.tgz#bbbff68978fefdbe68ccb533bc8cbe1d1afb5413" + integrity sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow== + +"@emotion/memoize@^0.7.4": + version "0.7.5" + resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.7.5.tgz#2c40f81449a4e554e9fc6396910ed4843ec2be50" + integrity sha512-igX9a37DR2ZPGYtV6suZ6whr8pTFtyHL3K/oLUotxpSVO2ASaprmAe2Dkq7tBo7CRY7MMDrAa9nuQP9/YG8FxQ== + +"@emotion/react@11.7.1": + version "11.7.1" + resolved "https://registry.yarnpkg.com/@emotion/react/-/react-11.7.1.tgz#3f800ce9b20317c13e77b8489ac4a0b922b2fe07" + integrity sha512-DV2Xe3yhkF1yT4uAUoJcYL1AmrnO5SVsdfvu+fBuS7IbByDeTVx9+wFmvx9Idzv7/78+9Mgx2Hcmr7Fex3tIyw== + dependencies: + "@babel/runtime" "^7.13.10" + "@emotion/cache" "^11.7.1" + "@emotion/serialize" "^1.0.2" + "@emotion/sheet" "^1.1.0" + "@emotion/utils" "^1.0.0" + "@emotion/weak-memoize" "^0.2.5" + hoist-non-react-statics "^3.3.1" + +"@emotion/serialize@1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@emotion/serialize/-/serialize-1.0.2.tgz#77cb21a0571c9f68eb66087754a65fa97bfcd965" + integrity sha512-95MgNJ9+/ajxU7QIAruiOAdYNjxZX7G2mhgrtDWswA21VviYIRP1R5QilZ/bDY42xiKsaktP4egJb3QdYQZi1A== + dependencies: + "@emotion/hash" "^0.8.0" + "@emotion/memoize" "^0.7.4" + "@emotion/unitless" "^0.7.5" + "@emotion/utils" "^1.0.0" + csstype "^3.0.2" + +"@emotion/serialize@^1.0.2": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@emotion/serialize/-/serialize-1.0.4.tgz#ff31fd11bb07999611199c2229e152faadc21a3c" + integrity sha512-1JHamSpH8PIfFwAMryO2bNka+y8+KA5yga5Ocf2d7ZEiJjb7xlLW7aknBGZqJLajuLOvJ+72vN+IBSwPlXD1Pg== + dependencies: + "@emotion/hash" "^0.8.0" + "@emotion/memoize" "^0.7.4" + "@emotion/unitless" "^0.7.5" + "@emotion/utils" "^1.0.0" + csstype "^3.0.2" + +"@emotion/sheet@^1.1.0", "@emotion/sheet@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@emotion/sheet/-/sheet-1.1.1.tgz#015756e2a9a3c7c5f11d8ec22966a8dbfbfac787" + integrity sha512-J3YPccVRMiTZxYAY0IOq3kd+hUP8idY8Kz6B/Cyo+JuXq52Ek+zbPbSQUrVQp95aJ+lsAW7DPL1P2Z+U1jGkKA== + +"@emotion/unitless@^0.7.5": + version "0.7.5" + resolved "https://registry.yarnpkg.com/@emotion/unitless/-/unitless-0.7.5.tgz#77211291c1900a700b8a78cfafda3160d76949ed" + integrity sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg== + +"@emotion/utils@1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@emotion/utils/-/utils-1.0.0.tgz#abe06a83160b10570816c913990245813a2fd6af" + integrity sha512-mQC2b3XLDs6QCW+pDQDiyO/EdGZYOygE8s5N5rrzjSI4M3IejPE/JPndCBwRT9z982aqQNi6beWs1UeayrQxxA== + +"@emotion/utils@^1.0.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@emotion/utils/-/utils-1.1.0.tgz#86b0b297f3f1a0f2bdb08eeac9a2f49afd40d0cf" + integrity sha512-iRLa/Y4Rs5H/f2nimczYmS5kFJEbpiVvgN3XVfZ022IYhuNA1IRSHEizcof88LtCTXtl9S2Cxt32KgaXEu72JQ== + +"@emotion/weak-memoize@^0.2.5": + version "0.2.5" + resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.2.5.tgz#8eed982e2ee6f7f4e44c253e12962980791efd46" + integrity sha512-6U71C2Wp7r5XtFtQzYrW5iKFT67OixrSxjI4MptCHzdSVlgabczzqLe0ZSgnub/5Kp4hSbpDB1tMytZY9pwxxA== + +"@eslint/eslintrc@^1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.3.0.tgz#29f92c30bb3e771e4a2048c95fa6855392dfac4f" + integrity sha512-UWW0TMTmk2d7hLcWD1/e2g5HDM/HQ3csaLSqXCfqwh4uNDuNqlaKWXmEsL4Cs41Z0KnILNvwbHAah3C2yt06kw== + dependencies: + ajv "^6.12.4" + debug "^4.3.2" + espree "^9.3.2" + globals "^13.15.0" + ignore "^5.2.0" + import-fresh "^3.2.1" + js-yaml "^4.1.0" + minimatch "^3.1.2" + strip-json-comments "^3.1.1" + +"@fortawesome/fontawesome-common-types@6.1.1": + version "6.1.1" + resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.1.1.tgz#7dc996042d21fc1ae850e3173b5c67b0549f9105" + integrity sha512-wVn5WJPirFTnzN6tR95abCx+ocH+3IFLXAgyavnf9hUmN0CfWoDjPT/BAWsUVwSlYYVBeCLJxaqi7ZGe4uSjBA== + +"@fortawesome/fontawesome-svg-core@^6": + version "6.1.1" + resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-svg-core/-/fontawesome-svg-core-6.1.1.tgz#3424ec6182515951816be9b11665d67efdce5b5f" + integrity sha512-NCg0w2YIp81f4V6cMGD9iomfsIj7GWrqmsa0ZsPh59G7PKiGN1KymZNxmF00ssuAlo/VZmpK6xazsGOwzKYUMg== + dependencies: + "@fortawesome/fontawesome-common-types" "6.1.1" + +"@fortawesome/free-regular-svg-icons@^6": + version "6.1.1" + resolved "https://registry.yarnpkg.com/@fortawesome/free-regular-svg-icons/-/free-regular-svg-icons-6.1.1.tgz#3f2f58262a839edf0643cbacee7a8a8230061c98" + integrity sha512-xXiW7hcpgwmWtndKPOzG+43fPH7ZjxOaoeyooptSztGmJxCAflHZxXNK0GcT0uEsR4jTGQAfGklDZE5NHoBhKg== + dependencies: + "@fortawesome/fontawesome-common-types" "6.1.1" + +"@fortawesome/free-solid-svg-icons@^6": + version "6.1.1" + resolved "https://registry.yarnpkg.com/@fortawesome/free-solid-svg-icons/-/free-solid-svg-icons-6.1.1.tgz#3369e673f8fe8be2fba30b1ec274d47490a830a6" + integrity sha512-0/5exxavOhI/D4Ovm2r3vxNojGZioPwmFrKg0ZUH69Q68uFhFPs6+dhAToh6VEQBntxPRYPuT5Cg1tpNa9JUPg== + dependencies: + "@fortawesome/fontawesome-common-types" "6.1.1" + +"@fortawesome/react-fontawesome@^0": + version "0.1.18" + resolved "https://registry.yarnpkg.com/@fortawesome/react-fontawesome/-/react-fontawesome-0.1.18.tgz#dae37f718a24e14d7a99a5496c873d69af3fbd73" + integrity sha512-RwLIB4TZw0M9gvy5u+TusAA0afbwM4JQIimNH/j3ygd6aIvYPQLqXMhC9ErY26J23rDPyDZldIfPq/HpTTJ/tQ== + dependencies: + prop-types "^15.8.1" + +"@humanwhocodes/config-array@^0.9.2": + version "0.9.5" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.9.5.tgz#2cbaf9a89460da24b5ca6531b8bbfc23e1df50c7" + integrity sha512-ObyMyWxZiCu/yTisA7uzx81s40xR2fD5Cg/2Kq7G02ajkNubJf6BopgDTmDyc3U7sXpNKM8cYOw7s7Tyr+DnCw== + dependencies: + "@humanwhocodes/object-schema" "^1.2.1" + debug "^4.1.1" + minimatch "^3.0.4" + +"@humanwhocodes/object-schema@^1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" + integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== + "@jridgewell/gen-mapping@^0.1.0": version "0.1.1" resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz#e5d2e450306a9491e3bd77e323e38d7aff315996" @@ -274,6 +435,129 @@ "@jridgewell/resolve-uri" "^3.0.3" "@jridgewell/sourcemap-codec" "^1.4.10" +"@mantine/core@^4": + version "4.2.10" + resolved "https://registry.yarnpkg.com/@mantine/core/-/core-4.2.10.tgz#6b4973bc5c79cd077341ab2bbe327749ca3ed8c4" + integrity sha512-UCPhDcumygfBvik64VkMnBvqy0ZN9q+1AQ0fPdK8aAUvjRBWSyH0dJPL55vsK1ODboKktSEsyjHtb09DroL7fA== + dependencies: + "@mantine/styles" "4.2.10" + "@popperjs/core" "^2.9.3" + "@radix-ui/react-scroll-area" "^0.1.1" + react-popper "^2.2.5" + react-textarea-autosize "^8.3.2" + +"@mantine/hooks@^4": + version "4.2.10" + resolved "https://registry.yarnpkg.com/@mantine/hooks/-/hooks-4.2.10.tgz#ad55d5ad3c5814eab924dfb6fd04f9ffd44e3d30" + integrity sha512-gVYWeE4Ieu6FBwh9h/3FjcrrNzKx1k6Yw07/LSngJP0uT3fLt1gvY2p4PtPpOh7z2/RpTXBR1x+dOgEUKomYUQ== + +"@mantine/styles@4.2.10": + version "4.2.10" + resolved "https://registry.yarnpkg.com/@mantine/styles/-/styles-4.2.10.tgz#967f0b09f4b3adaa23e650f9112be362ed07e953" + integrity sha512-dWwCzBLYE8CwPJSPlV3MyIuR4ET3u6O+T+VmxY0lab6w/bqflseNSh9UBIW2AUoIYtxPVDjbZfzeTw32aV8CxA== + dependencies: + "@emotion/cache" "11.7.1" + "@emotion/react" "11.7.1" + "@emotion/serialize" "1.0.2" + "@emotion/utils" "1.0.0" + clsx "^1.1.1" + csstype "3.0.9" + +"@popperjs/core@^2.9.3": + version "2.11.5" + resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.5.tgz#db5a11bf66bdab39569719555b0f76e138d7bd64" + integrity sha512-9X2obfABZuDVLCgPK9aX0a/x4jaOEweTTWE2+9sr0Qqqevj2Uv5XorvusThmc9XGYpS9yI+fhh8RTafBtGposw== + +"@radix-ui/number@0.1.0": + version "0.1.0" + resolved "https://registry.yarnpkg.com/@radix-ui/number/-/number-0.1.0.tgz#73ad13d5cc5f75fa5e147d72e5d5d5e50d688256" + integrity sha512-rpf6QiOWLHAkM4FEMYu9i+5Jr8cKT893+R4mPpcdsy4LD7omr9JfdOqj/h/xPA5+EcVrpMMlU6rrRYpUB5UI8g== + dependencies: + "@babel/runtime" "^7.13.10" + +"@radix-ui/primitive@0.1.0": + version "0.1.0" + resolved "https://registry.yarnpkg.com/@radix-ui/primitive/-/primitive-0.1.0.tgz#6206b97d379994f0d1929809db035733b337e543" + integrity sha512-tqxZKybwN5Fa3VzZry4G6mXAAb9aAqKmPtnVbZpL0vsBwvOHTBwsjHVPXylocYLwEtBY9SCe665bYnNB515uoA== + dependencies: + "@babel/runtime" "^7.13.10" + +"@radix-ui/react-compose-refs@0.1.0": + version "0.1.0" + resolved "https://registry.yarnpkg.com/@radix-ui/react-compose-refs/-/react-compose-refs-0.1.0.tgz#cff6e780a0f73778b976acff2c2a5b6551caab95" + integrity sha512-eyclbh+b77k+69Dk72q3694OHrn9B3QsoIRx7ywX341U9RK1ThgQjMFZoPtmZNQTksXHLNEiefR8hGVeFyInGg== + dependencies: + "@babel/runtime" "^7.13.10" + +"@radix-ui/react-context@0.1.1": + version "0.1.1" + resolved "https://registry.yarnpkg.com/@radix-ui/react-context/-/react-context-0.1.1.tgz#06996829ea124d9a1bc1dbe3e51f33588fab0875" + integrity sha512-PkyVX1JsLBioeu0jB9WvRpDBBLtLZohVDT3BB5CTSJqActma8S8030P57mWZb4baZifMvN7KKWPAA40UmWKkQg== + dependencies: + "@babel/runtime" "^7.13.10" + +"@radix-ui/react-presence@0.1.2": + version "0.1.2" + resolved "https://registry.yarnpkg.com/@radix-ui/react-presence/-/react-presence-0.1.2.tgz#9f11cce3df73cf65bc348e8b76d891f0d54c1fe3" + integrity sha512-3BRlFZraooIUfRlyN+b/Xs5hq1lanOOo/+3h6Pwu2GMFjkGKKa4Rd51fcqGqnVlbr3jYg+WLuGyAV4KlgqwrQw== + dependencies: + "@babel/runtime" "^7.13.10" + "@radix-ui/react-compose-refs" "0.1.0" + "@radix-ui/react-use-layout-effect" "0.1.0" + +"@radix-ui/react-primitive@0.1.4": + version "0.1.4" + resolved "https://registry.yarnpkg.com/@radix-ui/react-primitive/-/react-primitive-0.1.4.tgz#6c233cf08b0cb87fecd107e9efecb3f21861edc1" + integrity sha512-6gSl2IidySupIMJFjYnDIkIWRyQdbu/AHK7rbICPani+LW4b0XdxBXc46og/iZvuwW8pjCS8I2SadIerv84xYA== + dependencies: + "@babel/runtime" "^7.13.10" + "@radix-ui/react-slot" "0.1.2" + +"@radix-ui/react-scroll-area@^0.1.1": + version "0.1.4" + resolved "https://registry.yarnpkg.com/@radix-ui/react-scroll-area/-/react-scroll-area-0.1.4.tgz#be1d32c113ee9f64e3d2e7ee3983d98f00b42038" + integrity sha512-QHxRsjy+hsHwQYJ9cCNgSJ5+6ioZu1KhwD1UOXoHNciuFGMX08v+uJPKXIz+ySv03Rx6cOz6f/Fk5aPHRMFi/A== + dependencies: + "@babel/runtime" "^7.13.10" + "@radix-ui/number" "0.1.0" + "@radix-ui/primitive" "0.1.0" + "@radix-ui/react-compose-refs" "0.1.0" + "@radix-ui/react-context" "0.1.1" + "@radix-ui/react-presence" "0.1.2" + "@radix-ui/react-primitive" "0.1.4" + "@radix-ui/react-use-callback-ref" "0.1.0" + "@radix-ui/react-use-direction" "0.1.0" + "@radix-ui/react-use-layout-effect" "0.1.0" + +"@radix-ui/react-slot@0.1.2": + version "0.1.2" + resolved "https://registry.yarnpkg.com/@radix-ui/react-slot/-/react-slot-0.1.2.tgz#e6f7ad9caa8ce81cc8d532c854c56f9b8b6307c8" + integrity sha512-ADkqfL+agEzEguU3yS26jfB50hRrwf7U4VTwAOZEmi/g+ITcBWe12yM46ueS/UCIMI9Py+gFUaAdxgxafFvY2Q== + dependencies: + "@babel/runtime" "^7.13.10" + "@radix-ui/react-compose-refs" "0.1.0" + +"@radix-ui/react-use-callback-ref@0.1.0": + version "0.1.0" + resolved "https://registry.yarnpkg.com/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-0.1.0.tgz#934b6e123330f5b3a6b116460e6662cbc663493f" + integrity sha512-Va041McOFFl+aV+sejvl0BS2aeHx86ND9X/rVFmEFQKTXCp6xgUK0NGUAGcgBlIjnJSbMYPGEk1xKSSlVcN2Aw== + dependencies: + "@babel/runtime" "^7.13.10" + +"@radix-ui/react-use-direction@0.1.0": + version "0.1.0" + resolved "https://registry.yarnpkg.com/@radix-ui/react-use-direction/-/react-use-direction-0.1.0.tgz#97ac1d52e497c974389e7988f809238ed72e7df7" + integrity sha512-NajpY/An9TCPSfOVkgWIdXJV+VuWl67PxB6kOKYmtNAFHvObzIoh8o0n9sAuwSAyFCZVq211FEf9gvVDRhOyiA== + dependencies: + "@babel/runtime" "^7.13.10" + +"@radix-ui/react-use-layout-effect@0.1.0": + version "0.1.0" + resolved "https://registry.yarnpkg.com/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-0.1.0.tgz#ebf71bd6d2825de8f1fbb984abf2293823f0f223" + integrity sha512-+wdeS51Y+E1q1Wmd+1xSSbesZkpVj4jsg0BojCbopWvgq5iBvixw5vgemscdh58ep98BwUbsFYnrywFhV9yrVg== + dependencies: + "@babel/runtime" "^7.13.10" + "@rollup/pluginutils@^4.2.1": version "4.2.1" resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-4.2.1.tgz#e6c6c3aba0744edce3fb2074922d3776c0af2a6d" @@ -282,19 +566,29 @@ estree-walker "^2.0.1" picomatch "^2.2.2" +"@types/json5@^0.0.29": + version "0.0.29" + resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" + integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== + +"@types/minimatch@^3.0.3": + version "3.0.5" + resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40" + integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ== + "@types/prop-types@*": version "15.7.5" resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.5.tgz#5f19d2b85a98e9558036f6a3cacc8819420f05cf" integrity sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w== -"@types/react-dom@^18.0.0": +"@types/react-dom@^18": version "18.0.5" resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-18.0.5.tgz#330b2d472c22f796e5531446939eacef8378444a" integrity sha512-OWPWTUrY/NIrjsAPkAk1wW9LZeIjSvkXRhclsFO8CZcZGCOg2G0YZy4ft+rOyYxy8B7ui5iZzi9OkDebZ7/QSA== dependencies: "@types/react" "*" -"@types/react@*", "@types/react@^18.0.0": +"@types/react@*": version "18.0.13" resolved "https://registry.yarnpkg.com/@types/react/-/react-18.0.13.tgz#0f5bd24a5f26593e04e450fe85ff43f51c1524ff" integrity sha512-psqptIYQxGUFuGYwP3KCFVtPTkMpIcrqFmtKblWEUQhLuYLpHBwJkXhjp6eHfDM5IbyskY4x7qQpLedEsPkHlA== @@ -303,12 +597,21 @@ "@types/scheduler" "*" csstype "^3.0.2" +"@types/react@^18": + version "18.0.14" + resolved "https://registry.yarnpkg.com/@types/react/-/react-18.0.14.tgz#e016616ffff51dba01b04945610fe3671fdbe06d" + integrity sha512-x4gGuASSiWmo0xjDLpm5mPb52syZHJx02VKbqUKdLmKtAwIh63XClGsiTI1K6DO5q7ox4xAsQrU+Gl3+gGXF9Q== + dependencies: + "@types/prop-types" "*" + "@types/scheduler" "*" + csstype "^3.0.2" + "@types/scheduler@*": version "0.16.2" resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39" integrity sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew== -"@vitejs/plugin-react@^1.3.0": +"@vitejs/plugin-react@^1": version "1.3.2" resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-1.3.2.tgz#2fcf0b6ce9bcdcd4cec5c760c199779d5657ece1" integrity sha512-aurBNmMo0kz1O4qRoY+FM4epSA39y3ShWGuqfLRA/3z0oEJAdtoSfgA3aO98/PCCHAqMaduLxIxErWrVKIFzXA== @@ -322,6 +625,31 @@ react-refresh "^0.13.0" resolve "^1.22.0" +acorn-jsx@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" + integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== + +acorn@^8.7.1: + version "8.7.1" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30" + integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== + +ajv@^6.10.0, ajv@^6.12.4: + version "6.12.6" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" + integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== + dependencies: + fast-deep-equal "^3.1.1" + fast-json-stable-stringify "^2.0.0" + json-schema-traverse "^0.4.1" + uri-js "^4.2.2" + +ansi-regex@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== + ansi-styles@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" @@ -329,6 +657,77 @@ ansi-styles@^3.2.1: dependencies: color-convert "^1.9.0" +ansi-styles@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== + dependencies: + color-convert "^2.0.1" + +argparse@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" + integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== + +array-differ@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/array-differ/-/array-differ-3.0.0.tgz#3cbb3d0f316810eafcc47624734237d6aee4ae6b" + integrity sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg== + +array-includes@^3.1.4, array-includes@^3.1.5: + version "3.1.5" + resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.5.tgz#2c320010db8d31031fd2a5f6b3bbd4b1aad31bdb" + integrity sha512-iSDYZMMyTPkiFasVqfuAQnWAYcvO/SeBSCGKePoEthjp4LEMTe4uLc7b025o4jAZpHhihh8xPo99TNWUWWkGDQ== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.4" + es-abstract "^1.19.5" + get-intrinsic "^1.1.1" + is-string "^1.0.7" + +array-union@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" + integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== + +array.prototype.flat@^1.2.5: + version "1.3.0" + resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.3.0.tgz#0b0c1567bf57b38b56b4c97b8aa72ab45e4adc7b" + integrity sha512-12IUEkHsAhA4DY5s0FPgNXIdc8VRSqD9Zp78a5au9abH/SOBrsp082JOWFNTjkMozh8mqcdiKuaLGhPeYztxSw== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + es-abstract "^1.19.2" + es-shim-unscopables "^1.0.0" + +array.prototype.flatmap@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.3.0.tgz#a7e8ed4225f4788a70cd910abcf0791e76a5534f" + integrity sha512-PZC9/8TKAIxcWKdyeb77EzULHPrIX/tIZebLJUQOMR1OwYosT8yggdfWScfTBCDj5utONvOuPQQumYsU2ULbkg== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + es-abstract "^1.19.2" + es-shim-unscopables "^1.0.0" + +arrify@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" + integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== + +balanced-match@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== + +brace-expansion@^1.1.7: + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + browserslist@^4.20.2: version "4.20.4" resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.20.4.tgz#98096c9042af689ee1e0271333dbc564b8ce4477" @@ -340,6 +739,19 @@ browserslist@^4.20.2: node-releases "^2.0.5" picocolors "^1.0.0" +call-bind@^1.0.0, call-bind@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" + integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== + dependencies: + function-bind "^1.1.1" + get-intrinsic "^1.0.2" + +callsites@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" + integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== + caniuse-lite@^1.0.30001349: version "1.0.30001355" resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001355.tgz#e240b7177443ed0198c737a7f609536976701c77" @@ -354,6 +766,27 @@ chalk@^2.0.0: escape-string-regexp "^1.0.5" supports-color "^5.3.0" +chalk@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4" + integrity sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +chalk@^4.0.0: + version "4.1.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" + integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +clsx@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.1.1.tgz#98b3134f9abbdf23b2663491ace13c5c03a73188" + integrity sha512-6/bPho624p3S2pMyvP5kKBPXnI3ufHLObBFCfgx+LkeR5lg2XYy2hqZqUf45ypD8COn2bhgGJSUE+l5dhNBieA== + color-convert@^1.9.0: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" @@ -361,11 +794,28 @@ color-convert@^1.9.0: dependencies: color-name "1.1.3" +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== + dependencies: + color-name "~1.1.4" + color-name@1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== + convert-source-map@^1.7.0: version "1.8.0" resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.8.0.tgz#f3373c32d21b4d780dd8004514684fb791ca4369" @@ -373,23 +823,130 @@ convert-source-map@^1.7.0: dependencies: safe-buffer "~5.1.1" +cross-spawn@^7.0.0, cross-spawn@^7.0.2: + version "7.0.3" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" + integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + dependencies: + path-key "^3.1.0" + shebang-command "^2.0.0" + which "^2.0.1" + +csstype@3.0.9: + version "3.0.9" + resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.0.9.tgz#6410af31b26bd0520933d02cbc64fce9ce3fbf0b" + integrity sha512-rpw6JPxK6Rfg1zLOYCSwle2GFOOsnjmDYDaBwEcwoOg4qlsIVCN789VkBZDJAGi4T07gI4YSutR43t9Zz4Lzuw== + csstype@^3.0.2: version "3.1.0" resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.0.tgz#4ddcac3718d787cf9df0d1b7d15033925c8f29f2" integrity sha512-uX1KG+x9h5hIJsaKR9xHUeUraxf8IODOwq9JLNPq6BwB04a/xgpq3rcx47l5BZu5zBPlgD342tdke3Hom/nJRA== -debug@^4.1.0: +debug@^2.6.9: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + +debug@^3.2.7: + version "3.2.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" + integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== + dependencies: + ms "^2.1.1" + +debug@^4.1.0, debug@^4.1.1, debug@^4.3.2: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== dependencies: ms "2.1.2" +deep-is@^0.1.3: + version "0.1.4" + resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" + integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== + +define-properties@^1.1.3, define-properties@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.4.tgz#0b14d7bd7fbeb2f3572c3a7eda80ea5d57fb05b1" + integrity sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA== + dependencies: + has-property-descriptors "^1.0.0" + object-keys "^1.1.1" + +doctrine@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.1.0.tgz#5cd01fc101621b42c4cd7f5d1a66243716d3f39d" + integrity sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw== + dependencies: + esutils "^2.0.2" + +doctrine@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-3.0.0.tgz#addebead72a6574db783639dc87a121773973961" + integrity sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w== + dependencies: + esutils "^2.0.2" + electron-to-chromium@^1.4.147: version "1.4.158" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.158.tgz#abbdaaf64676bfa4bc0307522125db34424a0ada" integrity sha512-gppO3/+Y6sP432HtvwvuU8S+YYYLH4PmAYvQwqUtt9HDOmEsBwQfLnK9T8+1NIKwAS1BEygIjTaATC4H5EzvxQ== +end-of-stream@^1.1.0: + version "1.4.4" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" + integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== + dependencies: + once "^1.4.0" + +es-abstract@^1.19.0, es-abstract@^1.19.1, es-abstract@^1.19.2, es-abstract@^1.19.5: + version "1.20.1" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.20.1.tgz#027292cd6ef44bd12b1913b828116f54787d1814" + integrity sha512-WEm2oBhfoI2sImeM4OF2zE2V3BYdSF+KnSi9Sidz51fQHd7+JuF8Xgcj9/0o+OWeIeIS/MiuNnlruQrJf16GQA== + dependencies: + call-bind "^1.0.2" + es-to-primitive "^1.2.1" + function-bind "^1.1.1" + function.prototype.name "^1.1.5" + get-intrinsic "^1.1.1" + get-symbol-description "^1.0.0" + has "^1.0.3" + has-property-descriptors "^1.0.0" + has-symbols "^1.0.3" + internal-slot "^1.0.3" + is-callable "^1.2.4" + is-negative-zero "^2.0.2" + is-regex "^1.1.4" + is-shared-array-buffer "^1.0.2" + is-string "^1.0.7" + is-weakref "^1.0.2" + object-inspect "^1.12.0" + object-keys "^1.1.1" + object.assign "^4.1.2" + regexp.prototype.flags "^1.4.3" + string.prototype.trimend "^1.0.5" + string.prototype.trimstart "^1.0.5" + unbox-primitive "^1.0.2" + +es-shim-unscopables@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.0.0.tgz#702e632193201e3edf8713635d083d378e510241" + integrity sha512-Jm6GPcCdC30eMLbZ2x8z2WuRwAws3zTBBKuusffYVUrNj/GVSUAZ+xKMaUpfNDR5IbyNA5LJbaecoUVbmUcB1w== + dependencies: + has "^1.0.3" + +es-to-primitive@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a" + integrity sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA== + dependencies: + is-callable "^1.1.4" + is-date-object "^1.0.1" + is-symbol "^1.0.2" + esbuild-android-64@0.14.44: version "0.14.44" resolved "https://registry.yarnpkg.com/esbuild-android-64/-/esbuild-android-64-0.14.44.tgz#62f5cb563d0ba318d898b6eb230c61ad3dc93619" @@ -526,11 +1083,274 @@ escape-string-regexp@^1.0.5: resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== +escape-string-regexp@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" + integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== + +eslint-import-resolver-node@^0.3.6: + version "0.3.6" + resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz#4048b958395da89668252001dbd9eca6b83bacbd" + integrity sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw== + dependencies: + debug "^3.2.7" + resolve "^1.20.0" + +eslint-module-utils@^2.7.3: + version "2.7.3" + resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.7.3.tgz#ad7e3a10552fdd0642e1e55292781bd6e34876ee" + integrity sha512-088JEC7O3lDZM9xGe0RerkOMd0EjFl+Yvd1jPWIkMT5u3H9+HC34mWWPnqPrN13gieT9pBOO+Qt07Nb/6TresQ== + dependencies: + debug "^3.2.7" + find-up "^2.1.0" + +eslint-plugin-import@^2: + version "2.26.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz#f812dc47be4f2b72b478a021605a59fc6fe8b88b" + integrity sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA== + dependencies: + array-includes "^3.1.4" + array.prototype.flat "^1.2.5" + debug "^2.6.9" + doctrine "^2.1.0" + eslint-import-resolver-node "^0.3.6" + eslint-module-utils "^2.7.3" + has "^1.0.3" + is-core-module "^2.8.1" + is-glob "^4.0.3" + minimatch "^3.1.2" + object.values "^1.1.5" + resolve "^1.22.0" + tsconfig-paths "^3.14.1" + +eslint-plugin-prettier@^4: + version "4.0.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-4.0.0.tgz#8b99d1e4b8b24a762472b4567992023619cb98e0" + integrity sha512-98MqmCJ7vJodoQK359bqQWaxOE0CS8paAz/GgjaZLyex4TTk3g9HugoO89EqWCrFiOqn9EVvcoo7gZzONCWVwQ== + dependencies: + prettier-linter-helpers "^1.0.0" + +eslint-plugin-react-hooks@^4: + version "4.6.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.0.tgz#4c3e697ad95b77e93f8646aaa1630c1ba607edd3" + integrity sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g== + +eslint-plugin-react-perf@^3: + version "3.3.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-react-perf/-/eslint-plugin-react-perf-3.3.1.tgz#e52d64c1d6e9c4cb1240108cdbbe406837f1b887" + integrity sha512-iOx2UtEOH50TmQhezTS4jbBAj/2gbrUdX+ZM28c2K9mwTvtRX6gdnd2P4WPQrejITDsAMNTCz95zu5HcjCD0xg== + +eslint-plugin-react@^7: + version "7.30.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-react/-/eslint-plugin-react-7.30.0.tgz#8e7b1b2934b8426ac067a0febade1b13bd7064e3" + integrity sha512-RgwH7hjW48BleKsYyHK5vUAvxtE9SMPDKmcPRQgtRCYaZA0XQPt5FSkrU3nhz5ifzMZcA8opwmRJ2cmOO8tr5A== + dependencies: + array-includes "^3.1.5" + array.prototype.flatmap "^1.3.0" + doctrine "^2.1.0" + estraverse "^5.3.0" + jsx-ast-utils "^2.4.1 || ^3.0.0" + minimatch "^3.1.2" + object.entries "^1.1.5" + object.fromentries "^2.0.5" + object.hasown "^1.1.1" + object.values "^1.1.5" + prop-types "^15.8.1" + resolve "^2.0.0-next.3" + semver "^6.3.0" + string.prototype.matchall "^4.0.7" + +eslint-plugin-unused-imports@^2: + version "2.0.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-unused-imports/-/eslint-plugin-unused-imports-2.0.0.tgz#d8db8c4d0cfa0637a8b51ce3fd7d1b6bc3f08520" + integrity sha512-3APeS/tQlTrFa167ThtP0Zm0vctjr4M44HMpeg1P4bK6wItarumq0Ma82xorMKdFsWpphQBlRPzw/pxiVELX1A== + dependencies: + eslint-rule-composer "^0.3.0" + +eslint-rule-composer@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/eslint-rule-composer/-/eslint-rule-composer-0.3.0.tgz#79320c927b0c5c0d3d3d2b76c8b4a488f25bbaf9" + integrity sha512-bt+Sh8CtDmn2OajxvNO+BX7Wn4CIWMpTRm3MaiKPCQcnnlm0CS2mhui6QaoeQugs+3Kj2ESKEEGJUdVafwhiCg== + +eslint-scope@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.1.1.tgz#fff34894c2f65e5226d3041ac480b4513a163642" + integrity sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw== + dependencies: + esrecurse "^4.3.0" + estraverse "^5.2.0" + +eslint-utils@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-3.0.0.tgz#8aebaface7345bb33559db0a1f13a1d2d48c3672" + integrity sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA== + dependencies: + eslint-visitor-keys "^2.0.0" + +eslint-visitor-keys@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz#f65328259305927392c938ed44eb0a5c9b2bd303" + integrity sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw== + +eslint-visitor-keys@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" + integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== + +eslint@^8: + version "8.18.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.18.0.tgz#78d565d16c993d0b73968c523c0446b13da784fd" + integrity sha512-As1EfFMVk7Xc6/CvhssHUjsAQSkpfXvUGMFC3ce8JDe6WvqCgRrLOBQbVpsBFr1X1V+RACOadnzVvcUS5ni2bA== + dependencies: + "@eslint/eslintrc" "^1.3.0" + "@humanwhocodes/config-array" "^0.9.2" + ajv "^6.10.0" + chalk "^4.0.0" + cross-spawn "^7.0.2" + debug "^4.3.2" + doctrine "^3.0.0" + escape-string-regexp "^4.0.0" + eslint-scope "^7.1.1" + eslint-utils "^3.0.0" + eslint-visitor-keys "^3.3.0" + espree "^9.3.2" + esquery "^1.4.0" + esutils "^2.0.2" + fast-deep-equal "^3.1.3" + file-entry-cache "^6.0.1" + functional-red-black-tree "^1.0.1" + glob-parent "^6.0.1" + globals "^13.15.0" + ignore "^5.2.0" + import-fresh "^3.0.0" + imurmurhash "^0.1.4" + is-glob "^4.0.0" + js-yaml "^4.1.0" + json-stable-stringify-without-jsonify "^1.0.1" + levn "^0.4.1" + lodash.merge "^4.6.2" + minimatch "^3.1.2" + natural-compare "^1.4.0" + optionator "^0.9.1" + regexpp "^3.2.0" + strip-ansi "^6.0.1" + strip-json-comments "^3.1.0" + text-table "^0.2.0" + v8-compile-cache "^2.0.3" + +espree@^9.3.2: + version "9.3.2" + resolved "https://registry.yarnpkg.com/espree/-/espree-9.3.2.tgz#f58f77bd334731182801ced3380a8cc859091596" + integrity sha512-D211tC7ZwouTIuY5x9XnS0E9sWNChB7IYKX/Xp5eQj3nFXhqmiUDB9q27y76oFl8jTg3pXcQx/bpxMfs3CIZbA== + dependencies: + acorn "^8.7.1" + acorn-jsx "^5.3.2" + eslint-visitor-keys "^3.3.0" + +esquery@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.4.0.tgz#2148ffc38b82e8c7057dfed48425b3e61f0f24a5" + integrity sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w== + dependencies: + estraverse "^5.1.0" + +esrecurse@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/esrecurse/-/esrecurse-4.3.0.tgz#7ad7964d679abb28bee72cec63758b1c5d2c9921" + integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag== + dependencies: + estraverse "^5.2.0" + +estraverse@^5.1.0, estraverse@^5.2.0, estraverse@^5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123" + integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA== + estree-walker@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-2.0.2.tgz#52f010178c2a4c117a7757cfe942adb7d2da4cac" integrity sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w== +esutils@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" + integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== + +execa@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a" + integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA== + dependencies: + cross-spawn "^7.0.0" + get-stream "^5.0.0" + human-signals "^1.1.1" + is-stream "^2.0.0" + merge-stream "^2.0.0" + npm-run-path "^4.0.0" + onetime "^5.1.0" + signal-exit "^3.0.2" + strip-final-newline "^2.0.0" + +fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" + integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== + +fast-diff@^1.1.2: + version "1.2.0" + resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.2.0.tgz#73ee11982d86caaf7959828d519cfe927fac5f03" + integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w== + +fast-json-stable-stringify@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" + integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== + +fast-levenshtein@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" + integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== + +file-entry-cache@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" + integrity sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg== + dependencies: + flat-cache "^3.0.4" + +find-up@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-2.1.0.tgz#45d1b7e506c717ddd482775a2b77920a3c0c57a7" + integrity sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ== + dependencies: + locate-path "^2.0.0" + +find-up@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" + integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== + dependencies: + locate-path "^5.0.0" + path-exists "^4.0.0" + +flat-cache@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.0.4.tgz#61b0338302b2fe9f957dcc32fc2a87f1c3048b11" + integrity sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg== + dependencies: + flatted "^3.1.0" + rimraf "^3.0.2" + +flatted@^3.1.0: + version "3.2.5" + resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.5.tgz#76c8584f4fc843db64702a6bd04ab7a8bd666da3" + integrity sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg== + +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== + fsevents@~2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" @@ -541,21 +1361,120 @@ function-bind@^1.1.1: resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== +function.prototype.name@^1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.5.tgz#cce0505fe1ffb80503e6f9e46cc64e46a12a9621" + integrity sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + es-abstract "^1.19.0" + functions-have-names "^1.2.2" + +functional-red-black-tree@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327" + integrity sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g== + +functions-have-names@^1.2.2: + version "1.2.3" + resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834" + integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ== + gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg== +get-intrinsic@^1.0.2, get-intrinsic@^1.1.0, get-intrinsic@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.2.tgz#336975123e05ad0b7ba41f152ee4aadbea6cf598" + integrity sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA== + dependencies: + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.3" + +get-stream@^5.0.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" + integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== + dependencies: + pump "^3.0.0" + +get-symbol-description@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/get-symbol-description/-/get-symbol-description-1.0.0.tgz#7fdb81c900101fbd564dd5f1a30af5aadc1e58d6" + integrity sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw== + dependencies: + call-bind "^1.0.2" + get-intrinsic "^1.1.1" + +glob-parent@^6.0.1: + version "6.0.2" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-6.0.2.tgz#6d237d99083950c79290f24c7642a3de9a28f9e3" + integrity sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A== + dependencies: + is-glob "^4.0.3" + +glob@^7.1.3: + version "7.2.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" + integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.1.1" + once "^1.3.0" + path-is-absolute "^1.0.0" + globals@^11.1.0: version "11.12.0" resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== +globals@^13.15.0: + version "13.15.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-13.15.0.tgz#38113218c907d2f7e98658af246cef8b77e90bac" + integrity sha512-bpzcOlgDhMG070Av0Vy5Owklpv1I6+j96GhUI7Rh7IzDCKLzboflLrrfqMu8NquDbiR4EOQk7XzJwqVJxicxog== + dependencies: + type-fest "^0.20.2" + +has-bigints@^1.0.1, has-bigints@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa" + integrity sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ== + has-flag@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" integrity sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw== +has-flag@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== + +has-property-descriptors@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz#610708600606d36961ed04c196193b6a607fa861" + integrity sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ== + dependencies: + get-intrinsic "^1.1.1" + +has-symbols@^1.0.1, has-symbols@^1.0.2, has-symbols@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" + integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== + +has-tostringtag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.0.tgz#7e133818a7d394734f941e73c3d3f9291e658b25" + integrity sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ== + dependencies: + has-symbols "^1.0.2" + has@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" @@ -563,50 +1482,482 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" -is-core-module@^2.8.1: +history@^5.2.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/history/-/history-5.3.0.tgz#1548abaa245ba47992f063a0783db91ef201c73b" + integrity sha512-ZqaKwjjrAYUYfLG+htGaIIZ4nioX2L70ZUMIFysS3xvBsSG4x/n1V6TXV3N8ZYNuFGlDirFg32T7B6WOUPDYcQ== + dependencies: + "@babel/runtime" "^7.7.6" + +hoist-non-react-statics@^3.3.1: + version "3.3.2" + resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45" + integrity sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw== + dependencies: + react-is "^16.7.0" + +human-signals@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3" + integrity sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw== + +husky@^7: + version "7.0.4" + resolved "https://registry.yarnpkg.com/husky/-/husky-7.0.4.tgz#242048245dc49c8fb1bf0cc7cfb98dd722531535" + integrity sha512-vbaCKN2QLtP/vD4yvs6iz6hBEo6wkSzs8HpRah1Z6aGmF2KW5PdYuAd7uX5a+OyBZHBhd+TFLqgjUgytQr4RvQ== + +ignore@^5.1.4, ignore@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a" + integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ== + +import-fresh@^3.0.0, import-fresh@^3.2.1: + version "3.3.0" + resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" + integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== + dependencies: + parent-module "^1.0.0" + resolve-from "^4.0.0" + +imurmurhash@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" + integrity sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA== + +inflight@^1.0.4: + version "1.0.6" + resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + integrity sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA== + dependencies: + once "^1.3.0" + wrappy "1" + +inherits@2: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + +internal-slot@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.3.tgz#7347e307deeea2faac2ac6205d4bc7d34967f59c" + integrity sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA== + dependencies: + get-intrinsic "^1.1.0" + has "^1.0.3" + side-channel "^1.0.4" + +is-bigint@^1.0.1: + version "1.0.4" + resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.4.tgz#08147a1875bc2b32005d41ccd8291dffc6691df3" + integrity sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg== + dependencies: + has-bigints "^1.0.1" + +is-boolean-object@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.1.2.tgz#5c6dc200246dd9321ae4b885a114bb1f75f63719" + integrity sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA== + dependencies: + call-bind "^1.0.2" + has-tostringtag "^1.0.0" + +is-callable@^1.1.4, is-callable@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.4.tgz#47301d58dd0259407865547853df6d61fe471945" + integrity sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w== + +is-core-module@^2.8.1, is-core-module@^2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.9.0.tgz#e1c34429cd51c6dd9e09e0799e396e27b19a9c69" integrity sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A== dependencies: has "^1.0.3" +is-date-object@^1.0.1: + version "1.0.5" + resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.5.tgz#0841d5536e724c25597bf6ea62e1bd38298df31f" + integrity sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ== + dependencies: + has-tostringtag "^1.0.0" + +is-extglob@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" + integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ== + +is-glob@^4.0.0, is-glob@^4.0.3: + version "4.0.3" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" + integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== + dependencies: + is-extglob "^2.1.1" + +is-negative-zero@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.2.tgz#7bf6f03a28003b8b3965de3ac26f664d765f3150" + integrity sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA== + +is-number-object@^1.0.4: + version "1.0.7" + resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.7.tgz#59d50ada4c45251784e9904f5246c742f07a42fc" + integrity sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ== + dependencies: + has-tostringtag "^1.0.0" + +is-regex@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.4.tgz#eef5663cd59fa4c0ae339505323df6854bb15958" + integrity sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg== + dependencies: + call-bind "^1.0.2" + has-tostringtag "^1.0.0" + +is-shared-array-buffer@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz#8f259c573b60b6a32d4058a1a07430c0a7344c79" + integrity sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA== + dependencies: + call-bind "^1.0.2" + +is-stream@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" + integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== + +is-string@^1.0.5, is-string@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.7.tgz#0dd12bf2006f255bb58f695110eff7491eebc0fd" + integrity sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg== + dependencies: + has-tostringtag "^1.0.0" + +is-symbol@^1.0.2, is-symbol@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.4.tgz#a6dac93b635b063ca6872236de88910a57af139c" + integrity sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg== + dependencies: + has-symbols "^1.0.2" + +is-weakref@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-weakref/-/is-weakref-1.0.2.tgz#9529f383a9338205e89765e0392efc2f100f06f2" + integrity sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ== + dependencies: + call-bind "^1.0.2" + +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" + integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== + "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== +js-yaml@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== + dependencies: + argparse "^2.0.1" + jsesc@^2.5.1: version "2.5.2" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== +json-schema-traverse@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" + integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== + +json-stable-stringify-without-jsonify@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" + integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw== + +json5@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.1.tgz#779fb0018604fa854eacbf6252180d83543e3dbe" + integrity sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow== + dependencies: + minimist "^1.2.0" + json5@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.1.tgz#655d50ed1e6f95ad1a3caababd2b0efda10b395c" integrity sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA== -loose-envify@^1.1.0: +"jsx-ast-utils@^2.4.1 || ^3.0.0": + version "3.3.0" + resolved "https://registry.yarnpkg.com/jsx-ast-utils/-/jsx-ast-utils-3.3.0.tgz#e624f259143b9062c92b6413ff92a164c80d3ccb" + integrity sha512-XzO9luP6L0xkxwhIJMTJQpZo/eeN60K08jHdexfD569AGxeNug6UketeHXEhROoM8aR7EcUoOQmIhcJQjcuq8Q== + dependencies: + array-includes "^3.1.4" + object.assign "^4.1.2" + +levn@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" + integrity sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ== + dependencies: + prelude-ls "^1.2.1" + type-check "~0.4.0" + +locate-path@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e" + integrity sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA== + dependencies: + p-locate "^2.0.0" + path-exists "^3.0.0" + +locate-path@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0" + integrity sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g== + dependencies: + p-locate "^4.1.0" + +lodash.merge@^4.6.2: + version "4.6.2" + resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" + integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== + +loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" integrity sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q== dependencies: js-tokens "^3.0.0 || ^4.0.0" +merge-stream@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" + integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== + +mimic-fn@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" + integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== + +minimatch@^3.0.4, minimatch@^3.1.1, minimatch@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" + integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== + dependencies: + brace-expansion "^1.1.7" + +minimist@^1.2.0, minimist@^1.2.6: + version "1.2.6" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" + integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== + +mri@^1.1.5: + version "1.2.0" + resolved "https://registry.yarnpkg.com/mri/-/mri-1.2.0.tgz#6721480fec2a11a4889861115a48b6cbe7cc8f0b" + integrity sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA== + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A== + ms@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== +ms@^2.1.1: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + +multimatch@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/multimatch/-/multimatch-4.0.0.tgz#8c3c0f6e3e8449ada0af3dd29efb491a375191b3" + integrity sha512-lDmx79y1z6i7RNx0ZGCPq1bzJ6ZoDDKbvh7jxr9SJcWLkShMzXrHbYVpTdnhNM5MXpDUxCQ4DgqVttVXlBgiBQ== + dependencies: + "@types/minimatch" "^3.0.3" + array-differ "^3.0.0" + array-union "^2.1.0" + arrify "^2.0.1" + minimatch "^3.0.4" + nanoid@^3.3.4: version "3.3.4" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab" integrity sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw== +natural-compare@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" + integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== + node-releases@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.5.tgz#280ed5bc3eba0d96ce44897d8aee478bfb3d9666" integrity sha512-U9h1NLROZTq9uE1SNffn6WuPDg8icmi3ns4rEl/oTfIle4iLjTliCzgTsbaIFMq/Xn078/lfY/BL0GWZ+psK4Q== +npm-run-path@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea" + integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw== + dependencies: + path-key "^3.0.0" + +object-assign@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== + +object-inspect@^1.12.0, object-inspect@^1.9.0: + version "1.12.2" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" + integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== + +object-keys@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" + integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== + +object.assign@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.2.tgz#0ed54a342eceb37b38ff76eb831a0e788cb63940" + integrity sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ== + dependencies: + call-bind "^1.0.0" + define-properties "^1.1.3" + has-symbols "^1.0.1" + object-keys "^1.1.1" + +object.entries@^1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.5.tgz#e1acdd17c4de2cd96d5a08487cfb9db84d881861" + integrity sha512-TyxmjUoZggd4OrrU1W66FMDG6CuqJxsFvymeyXI51+vQLN67zYfZseptRge703kKQdo4uccgAKebXFcRCzk4+g== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + es-abstract "^1.19.1" + +object.fromentries@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.5.tgz#7b37b205109c21e741e605727fe8b0ad5fa08251" + integrity sha512-CAyG5mWQRRiBU57Re4FKoTBjXfDoNwdFVH2Y1tS9PqCsfUTymAohOkEMSG3aRNKmv4lV3O7p1et7c187q6bynw== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + es-abstract "^1.19.1" + +object.hasown@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/object.hasown/-/object.hasown-1.1.1.tgz#ad1eecc60d03f49460600430d97f23882cf592a3" + integrity sha512-LYLe4tivNQzq4JdaWW6WO3HMZZJWzkkH8fnI6EebWl0VZth2wL2Lovm74ep2/gZzlaTdV62JZHEqHQ2yVn8Q/A== + dependencies: + define-properties "^1.1.4" + es-abstract "^1.19.5" + +object.values@^1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.5.tgz#959f63e3ce9ef108720333082131e4a459b716ac" + integrity sha512-QUZRW0ilQ3PnPpbNtgdNV1PDbEqLIiSFB3l+EnGtBQ/8SUTLj1PZwtQHABZtLgwpJZTSZhuGLOGk57Drx2IvYg== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + es-abstract "^1.19.1" + +once@^1.3.0, once@^1.3.1, once@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== + dependencies: + wrappy "1" + +onetime@^5.1.0: + version "5.1.2" + resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e" + integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== + dependencies: + mimic-fn "^2.1.0" + +optionator@^0.9.1: + version "0.9.1" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" + integrity sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw== + dependencies: + deep-is "^0.1.3" + fast-levenshtein "^2.0.6" + levn "^0.4.1" + prelude-ls "^1.2.1" + type-check "^0.4.0" + word-wrap "^1.2.3" + +p-limit@^1.1.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.3.0.tgz#b86bd5f0c25690911c7590fcbfc2010d54b3ccb8" + integrity sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q== + dependencies: + p-try "^1.0.0" + +p-limit@^2.2.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" + integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== + dependencies: + p-try "^2.0.0" + +p-locate@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" + integrity sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg== + dependencies: + p-limit "^1.1.0" + +p-locate@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-4.1.0.tgz#a3428bb7088b3a60292f66919278b7c297ad4f07" + integrity sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A== + dependencies: + p-limit "^2.2.0" + +p-try@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" + integrity sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww== + +p-try@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" + integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== + +parent-module@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" + integrity sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g== + dependencies: + callsites "^3.0.0" + +path-exists@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" + integrity sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ== + +path-exists@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" + integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== + +path-is-absolute@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== + +path-key@^3.0.0, path-key@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" + integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== + path-parse@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" @@ -631,7 +1982,58 @@ postcss@^8.4.13: picocolors "^1.0.0" source-map-js "^1.0.2" -react-dom@^18.0.0: +prelude-ls@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" + integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== + +prettier-linter-helpers@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz#d23d41fe1375646de2d0104d3454a3008802cf7b" + integrity sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w== + dependencies: + fast-diff "^1.1.2" + +prettier@2: + version "2.7.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.7.1.tgz#e235806850d057f97bb08368a4f7d899f7760c64" + integrity sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g== + +pretty-quick@^3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/pretty-quick/-/pretty-quick-3.1.3.tgz#15281108c0ddf446675157ca40240099157b638e" + integrity sha512-kOCi2FJabvuh1as9enxYmrnBC6tVMoVOenMaBqRfsvBHB0cbpYHjdQEpSglpASDFEXVwplpcGR4CLEaisYAFcA== + dependencies: + chalk "^3.0.0" + execa "^4.0.0" + find-up "^4.1.0" + ignore "^5.1.4" + mri "^1.1.5" + multimatch "^4.0.0" + +prop-types@^15.8.1: + version "15.8.1" + resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5" + integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg== + dependencies: + loose-envify "^1.4.0" + object-assign "^4.1.1" + react-is "^16.13.1" + +pump@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" + integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww== + dependencies: + end-of-stream "^1.1.0" + once "^1.3.1" + +punycode@^2.1.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" + integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== + +react-dom@^18: version "18.2.0" resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-18.2.0.tgz#22aaf38708db2674ed9ada224ca4aa708d821e3d" integrity sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g== @@ -639,18 +2041,93 @@ react-dom@^18.0.0: loose-envify "^1.1.0" scheduler "^0.23.0" +react-fast-compare@^3.0.1: + version "3.2.0" + resolved "https://registry.yarnpkg.com/react-fast-compare/-/react-fast-compare-3.2.0.tgz#641a9da81b6a6320f270e89724fb45a0b39e43bb" + integrity sha512-rtGImPZ0YyLrscKI9xTpV8psd6I8VAtjKCzQDlzyDvqJA8XOW78TXYQwNRNd8g8JZnDu8q9Fu/1v4HPAVwVdHA== + +react-is@^16.13.1, react-is@^16.7.0: + version "16.13.1" + resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4" + integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ== + +react-popper@^2.2.5: + version "2.3.0" + resolved "https://registry.yarnpkg.com/react-popper/-/react-popper-2.3.0.tgz#17891c620e1320dce318bad9fede46a5f71c70ba" + integrity sha512-e1hj8lL3uM+sgSR4Lxzn5h1GxBlpa4CQz0XLF8kx4MDrDRWY0Ena4c97PUeSX9i5W3UAfDP0z0FXCTQkoXUl3Q== + dependencies: + react-fast-compare "^3.0.1" + warning "^4.0.2" + react-refresh@^0.13.0: version "0.13.0" resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.13.0.tgz#cbd01a4482a177a5da8d44c9755ebb1f26d5a1c1" integrity sha512-XP8A9BT0CpRBD+NYLLeIhld/RqG9+gktUjW1FkE+Vm7OCinbG1SshcK5tb9ls4kzvjZr9mOQc7HYgBngEyPAXg== -react@^18.0.0: +react-router-dom@^6: + version "6.3.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.3.0.tgz#a0216da813454e521905b5fa55e0e5176123f43d" + integrity sha512-uaJj7LKytRxZNQV8+RbzJWnJ8K2nPsOOEuX7aQstlMZKQT0164C+X2w6bnkqU3sjtLvpd5ojrezAyfZ1+0sStw== + dependencies: + history "^5.2.0" + react-router "6.3.0" + +react-router@6.3.0: + version "6.3.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.3.0.tgz#3970cc64b4cb4eae0c1ea5203a80334fdd175557" + integrity sha512-7Wh1DzVQ+tlFjkeo+ujvjSqSJmkt1+8JO+T5xklPlgrh70y7ogx75ODRW0ThWhY7S+6yEDks8TYrtQe/aoboBQ== + dependencies: + history "^5.2.0" + +react-textarea-autosize@^8.3.2: + version "8.3.4" + resolved "https://registry.yarnpkg.com/react-textarea-autosize/-/react-textarea-autosize-8.3.4.tgz#270a343de7ad350534141b02c9cb78903e553524" + integrity sha512-CdtmP8Dc19xL8/R6sWvtknD/eCXkQr30dtvC4VmGInhRsfF8X/ihXCq6+9l9qbxmKRiq407/7z5fxE7cVWQNgQ== + dependencies: + "@babel/runtime" "^7.10.2" + use-composed-ref "^1.3.0" + use-latest "^1.2.1" + +react@^18: version "18.2.0" resolved "https://registry.yarnpkg.com/react/-/react-18.2.0.tgz#555bd98592883255fa00de14f1151a917b5d77d5" integrity sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ== dependencies: loose-envify "^1.1.0" +regenerator-runtime@^0.13.4: + version "0.13.9" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz#8925742a98ffd90814988d7566ad30ca3b263b52" + integrity sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA== + +regexp.prototype.flags@^1.4.1, regexp.prototype.flags@^1.4.3: + version "1.4.3" + resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz#87cab30f80f66660181a3bb7bf5981a872b367ac" + integrity sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + functions-have-names "^1.2.2" + +regexpp@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.2.0.tgz#0425a2768d8f23bad70ca4b90461fa2f1213e1b2" + integrity sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg== + +resolve-from@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" + integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== + +resolve@^1.20.0: + version "1.22.1" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.1.tgz#27cb2ebb53f91abb49470a928bba7558066ac177" + integrity sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw== + dependencies: + is-core-module "^2.9.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + resolve@^1.22.0: version "1.22.0" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.0.tgz#5e0b8c67c15df57a89bdbabe603a002f21731198" @@ -660,6 +2137,22 @@ resolve@^1.22.0: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" +resolve@^2.0.0-next.3: + version "2.0.0-next.4" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-2.0.0-next.4.tgz#3d37a113d6429f496ec4752d2a2e58efb1fd4660" + integrity sha512-iMDbmAWtfU+MHpxt/I5iWI7cY6YVEZUQ3MBgPQ++XD1PELuJHIl82xBmObyP2KyQmkNB2dsqF7seoQQiAn5yDQ== + dependencies: + is-core-module "^2.9.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + +rimraf@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" + integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== + dependencies: + glob "^7.1.3" + rollup@^2.59.0: version "2.75.6" resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.75.6.tgz#ac4dc8600f95942a0180f61c7c9d6200e374b439" @@ -684,11 +2177,96 @@ semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +shebang-command@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" + integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== + dependencies: + shebang-regex "^3.0.0" + +shebang-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" + integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== + +side-channel@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" + integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== + dependencies: + call-bind "^1.0.0" + get-intrinsic "^1.0.2" + object-inspect "^1.9.0" + +signal-exit@^3.0.2: + version "3.0.7" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" + integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== + source-map-js@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw== +string.prototype.matchall@^4.0.7: + version "4.0.7" + resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.7.tgz#8e6ecb0d8a1fb1fda470d81acecb2dba057a481d" + integrity sha512-f48okCX7JiwVi1NXCVWcFnZgADDC/n2vePlQ/KUCNqCikLLilQvwjMO8+BHVKvgzH0JB0J9LEPgxOGT02RoETg== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + es-abstract "^1.19.1" + get-intrinsic "^1.1.1" + has-symbols "^1.0.3" + internal-slot "^1.0.3" + regexp.prototype.flags "^1.4.1" + side-channel "^1.0.4" + +string.prototype.trimend@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.5.tgz#914a65baaab25fbdd4ee291ca7dde57e869cb8d0" + integrity sha512-I7RGvmjV4pJ7O3kdf+LXFpVfdNOxtCW/2C8f6jNiW4+PQchwxkCDzlk1/7p+Wl4bqFIZeF47qAHXLuHHWKAxog== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.4" + es-abstract "^1.19.5" + +string.prototype.trimstart@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.5.tgz#5466d93ba58cfa2134839f81d7f42437e8c01fef" + integrity sha512-THx16TJCGlsN0o6dl2o6ncWUsdgnLRSA23rRE5pyGBw/mLr3Ej/R2LaqCtgP8VNMGZsvMWnf9ooZPyY2bHvUFg== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.4" + es-abstract "^1.19.5" + +strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +strip-bom@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" + integrity sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA== + +strip-final-newline@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad" + integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== + +strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" + integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== + +stylis@4.0.13: + version "4.0.13" + resolved "https://registry.yarnpkg.com/stylis/-/stylis-4.0.13.tgz#f5db332e376d13cc84ecfe5dace9a2a51d954c91" + integrity sha512-xGPXiFVl4YED9Jh7Euv2V220mriG9u4B2TA6Ybjc1catrstKD2PpIdU3U0RKpkVBC2EhmL/F0sPCr9vrFTNRag== + supports-color@^5.3.0: version "5.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" @@ -696,17 +2274,90 @@ supports-color@^5.3.0: dependencies: has-flag "^3.0.0" +supports-color@^7.1.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" + integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== + dependencies: + has-flag "^4.0.0" + supports-preserve-symlinks-flag@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== +text-table@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" + integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== + to-fast-properties@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" integrity sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog== -vite@^2.9.9: +tsconfig-paths@^3.14.1: + version "3.14.1" + resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz#ba0734599e8ea36c862798e920bcf163277b137a" + integrity sha512-fxDhWnFSLt3VuTwtvJt5fpwxBHg5AdKWMsgcPOOIilyjymcYVZoCQF8fvFRezCNfblEXmi+PcM1eYHeOAgXCOQ== + dependencies: + "@types/json5" "^0.0.29" + json5 "^1.0.1" + minimist "^1.2.6" + strip-bom "^3.0.0" + +type-check@^0.4.0, type-check@~0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" + integrity sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew== + dependencies: + prelude-ls "^1.2.1" + +type-fest@^0.20.2: + version "0.20.2" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" + integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== + +unbox-primitive@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.2.tgz#29032021057d5e6cdbd08c5129c226dff8ed6f9e" + integrity sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw== + dependencies: + call-bind "^1.0.2" + has-bigints "^1.0.2" + has-symbols "^1.0.3" + which-boxed-primitive "^1.0.2" + +uri-js@^4.2.2: + version "4.4.1" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" + integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== + dependencies: + punycode "^2.1.0" + +use-composed-ref@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/use-composed-ref/-/use-composed-ref-1.3.0.tgz#3d8104db34b7b264030a9d916c5e94fbe280dbda" + integrity sha512-GLMG0Jc/jiKov/3Ulid1wbv3r54K9HlMW29IWcDFPEqFkSO2nS0MuefWgMJpeHQ9YJeXDL3ZUF+P3jdXlZX/cQ== + +use-isomorphic-layout-effect@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.1.2.tgz#497cefb13d863d687b08477d9e5a164ad8c1a6fb" + integrity sha512-49L8yCO3iGT/ZF9QttjwLF/ZD9Iwto5LnH5LmEdk/6cFmXddqi2ulF0edxTwjj+7mqvpVVGQWvbXZdn32wRSHA== + +use-latest@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/use-latest/-/use-latest-1.2.1.tgz#d13dfb4b08c28e3e33991546a2cee53e14038cf2" + integrity sha512-xA+AVm/Wlg3e2P/JiItTziwS7FK92LWrDB0p+hgXloIMuVCeJJ8v6f0eeHyPZaJrM+usM1FkFfbNCrJGs8A/zw== + dependencies: + use-isomorphic-layout-effect "^1.1.1" + +v8-compile-cache@^2.0.3: + version "2.3.0" + resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" + integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== + +vite@^2: version "2.9.12" resolved "https://registry.yarnpkg.com/vite/-/vite-2.9.12.tgz#b1d636b0a8ac636afe9d83e3792d4895509a941b" integrity sha512-suxC36dQo9Rq1qMB2qiRorNJtJAdxguu5TMvBHOc/F370KvqAe9t48vYp+/TbPKRNrMh/J55tOUmkuIqstZaew== @@ -717,3 +2368,38 @@ vite@^2.9.9: rollup "^2.59.0" optionalDependencies: fsevents "~2.3.2" + +warning@^4.0.2: + version "4.0.3" + resolved "https://registry.yarnpkg.com/warning/-/warning-4.0.3.tgz#16e9e077eb8a86d6af7d64aa1e05fd85b4678ca3" + integrity sha512-rpJyN222KWIvHJ/F53XSZv0Zl/accqHR8et1kpaMTD/fLCRxtV8iX8czMzY7sVZupTI3zcUTg8eycS2kNF9l6w== + dependencies: + loose-envify "^1.0.0" + +which-boxed-primitive@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" + integrity sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg== + dependencies: + is-bigint "^1.0.1" + is-boolean-object "^1.1.0" + is-number-object "^1.0.4" + is-string "^1.0.5" + is-symbol "^1.0.3" + +which@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" + integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== + dependencies: + isexe "^2.0.0" + +word-wrap@^1.2.3: + version "1.2.3" + resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" + integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== + +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 4197b350df1..5084d6b893f 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -42,7 +42,7 @@ <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.46.v20220331</jetty.version> + <jetty.version>9.4.48.v20220622</jetty.version> <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> <org.lz4.version>1.8.0</org.lz4.version> <org.json.version>20090211</org.json.version> diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 61a946e9880..7611376fffb 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -82,6 +82,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusReplyThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}) default double feedNiceness() { return 0.0; } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultPoolNumThreads() { return 2; } @ModelFeatureFlag(owners = {"baldersheim"}) default int availableProcessors() { return 2; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxUnCommittedMemory() { return 130000; } diff --git a/config-model/CMakeLists.txt b/config-model/CMakeLists.txt index ffa896d8524..834830587d1 100644 --- a/config-model/CMakeLists.txt +++ b/config-model/CMakeLists.txt @@ -3,7 +3,7 @@ install_jar(config-model-jar-with-dependencies.jar) vespa_install_script(src/main/perl/vespa-deploy bin) -install(DIRECTORY src/main/resources/schema DESTINATION share/vespa PATTERN ".gitignore" EXCLUDE) -install(DIRECTORY src/main/resources/schema DESTINATION share/vespa/schema/version/8.x PATTERN ".gitignore" EXCLUDE) +install(DIRECTORY src/main/resources/schema DESTINATION share/vespa PATTERN ".gitignore" EXCLUDE PATTERN "version" EXCLUDE) +install(DIRECTORY src/main/resources/schema DESTINATION share/vespa/schema/version/8.x PATTERN ".gitignore" EXCLUDE PATTERN "version" EXCLUDE) # TODO: Remove when Vespa > 8 and no apps are left on 7 in hosted Vespa install(DIRECTORY src/main/resources/schema/version/7.x/ DESTINATION share/vespa/schema/version/7.x/schema PATTERN ".gitignore" EXCLUDE) diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 3e12b5c86e3..3da85a83119 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -50,6 +50,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private Quota quota = Quota.unlimited(); private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; + private double feedNiceness = 0.0; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; @@ -74,7 +75,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean persistenceThrottlingOfMergeFeedOps = true; private boolean useV8GeoPositions = true; private List<String> environmentVariables = List.of(); - private boolean avoidRenamingSummaryFeatures = true; private boolean enableBitVectors = false; private boolean loadCodeAsHugePages = false; private boolean sharedStringRepoNoReclaim = false; @@ -107,6 +107,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Quota quota() { return quota; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } + @Override public double feedNiceness() { return feedNiceness; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public List<TenantSecretStore> tenantSecretStores() { return tenantSecretStores; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @@ -131,7 +132,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean persistenceThrottlingOfMergeFeedOps() { return persistenceThrottlingOfMergeFeedOps; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public List<String> environmentVariables() { return environmentVariables; } - @Override public boolean avoidRenamingSummaryFeatures() { return this.avoidRenamingSummaryFeatures; } @Override public boolean enableBitVectors() { return this.enableBitVectors; } @Override public Architecture adminClusterArchitecture() { return adminClusterNodeResourcesArchitecture; } @Override public boolean sharedStringRepoNoReclaim() { return sharedStringRepoNoReclaim; } @@ -166,6 +166,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setFeedNiceness(double feedNiceness) { + this.feedNiceness = feedNiceness; + return this; + } + public TestProperties setAsyncMessageHandlingOnSchedule(boolean value) { useAsyncMessageHandlingOnSchedule = value; return this; @@ -351,11 +356,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setAvoidRenamingSummaryFeatures(boolean value) { - this.avoidRenamingSummaryFeatures = value; - return this; - } - public TestProperties setEnableBitVectors(boolean value) { this.enableBitVectors = value; return this; diff --git a/config-model/src/main/java/com/yahoo/schema/Application.java b/config-model/src/main/java/com/yahoo/schema/Application.java index b147388747e..71c0e563c77 100644 --- a/config-model/src/main/java/com/yahoo/schema/Application.java +++ b/config-model/src/main/java/com/yahoo/schema/Application.java @@ -54,8 +54,6 @@ public class Application { if (validate) schemas.forEach(schema -> schema.validate(logger)); - new TemporarySDTypeResolver(schemas, logger).process(); - List<SDDocumentType> sdocs = new ArrayList<>(); sdocs.add(SDDocumentType.VESPA_DOCUMENT); for (Schema schema : schemas) { @@ -64,9 +62,6 @@ public class Application { } } - var orderer = new SDDocumentTypeOrderer(sdocs, logger); - orderer.process(); - var resolver = new DocumentReferenceResolver(schemas); sdocs.forEach(resolver::resolveReferences); sdocs.forEach(resolver::resolveInheritedReferences); diff --git a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java index 29482431aa0..d98709569b1 100644 --- a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java +++ b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java @@ -132,8 +132,11 @@ public class ApplicationBuilder { this.deployLogger = deployLogger; this.properties = properties; this.documentsOnly = documentsOnly; - for (NamedReader reader : applicationPackage.getSchemas()) + var list = new ArrayList<>(applicationPackage.getSchemas()); + list.sort((a, b) -> a.getName().compareTo(b.getName())); + for (NamedReader reader : list) { addSchema(reader); + } } /** @@ -407,8 +410,13 @@ public class ApplicationBuilder { properties, rankProfileRegistry, queryProfileRegistry); - for (var i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { - builder.addSchemaFile(i.next().toString()); + + var fnli = Files.list(new File(dir).toPath()) + .map(p -> p.toString()) + .filter(fn -> fn.endsWith(".sd")) + .sorted(); + for (var i = fnli.iterator(); i.hasNext(); ) { + builder.addSchemaFile(i.next()); } builder.build(true); return builder; diff --git a/config-model/src/main/java/com/yahoo/schema/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/schema/SDDocumentTypeOrderer.java deleted file mode 100644 index 64bca9367d8..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/SDDocumentTypeOrderer.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.*; -import com.yahoo.document.annotation.AnnotationReferenceDataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.schema.document.SDDocumentType; -import com.yahoo.schema.document.TemporarySDDocumentType; - -import java.util.*; -import java.util.logging.Level; - -/** - * @author Einar M R Rosenvinge - */ -public class SDDocumentTypeOrderer { - - private final Map<DataTypeName, SDDocumentType> createdSDTypes = new LinkedHashMap<>(); - private final Set<Object> seenTypes = Collections.newSetFromMap(new IdentityHashMap<>()); - List<SDDocumentType> processingOrder = new LinkedList<>(); - private final DeployLogger deployLogger; - - public SDDocumentTypeOrderer(List<SDDocumentType> sdTypes, DeployLogger deployLogger) { - this.deployLogger = deployLogger; - for (SDDocumentType type : sdTypes) { - createdSDTypes.put(type.getDocumentName(), type); - } - } - - List<SDDocumentType> getOrdered() { return processingOrder; } - - public void process() { - for (SDDocumentType type : createdSDTypes.values()) { - process(type, type); - } - } - - private void process(SDDocumentType docOrStruct, SDDocumentType owningDocument) { - resolveAndProcessInheritedTemporaryTypes(docOrStruct, owningDocument); - if (seenTypes.contains(docOrStruct)) { - return; - } - seenTypes.add(docOrStruct); - for (Field field : docOrStruct.fieldSet()) { - var type = field.getDataType(); - String typeName = type.getName(); - if (!seenTypes.contains(type)) { - seenTypes.add(type); - //we haven't seen this before, do it - visit(type, owningDocument); - } - } - processingOrder.add(docOrStruct); - } - - private void resolveAndProcessInheritedTemporaryTypes(SDDocumentType type, SDDocumentType owningDocument) { - List<DataTypeName> toReplace = new ArrayList<>(); - for (SDDocumentType sdoc : type.getInheritedTypes()) { - if (sdoc instanceof TemporarySDDocumentType) { - toReplace.add(sdoc.getDocumentName()); - } - } - for (DataTypeName name : toReplace) { - SDDocumentType inherited; - if (type.isStruct()) { - inherited = owningDocument.allTypes().get(new NewDocumentType.Name(name.getName())); - if (inherited == null) throw new IllegalArgumentException("Struct '" + name + "' not found in " + owningDocument); - process(inherited, owningDocument); - } - else { - inherited = createdSDTypes.get(name); - if (inherited == null) { - throw new IllegalArgumentException("document " + type.getName() + - " inherits from unavailable document " + name); - } - process(inherited, inherited); - } - type.inherit(inherited); - } - } - - private SDDocumentType find(String name) { - SDDocumentType sdDocType = createdSDTypes.get(new DataTypeName(name)); - if (sdDocType != null) { - return sdDocType; - } - for(SDDocumentType sdoc : createdSDTypes.values()) { - for (SDDocumentType stype : sdoc.getTypes()) { - if (stype.getName().equals(name)) { - return stype; - } - } - } - return null; - } - - private void visit(DataType type, SDDocumentType owningDocument) { - if (type instanceof StructuredDataType) { - StructuredDataType structType = (StructuredDataType) type; - SDDocumentType sdDocType = owningDocument.getType(structType.getName()); - if (sdDocType == null) { - sdDocType = find(structType.getName()); - } - if (sdDocType == null) { - throw new IllegalArgumentException("Could not find struct '" + type.getName() + "'"); - } - process(sdDocType, owningDocument); - return; - } - - if (type instanceof MapDataType) { - MapDataType mType = (MapDataType) type; - visit(mType.getValueType(), owningDocument); - visit(mType.getKeyType(), owningDocument); - } else if (type instanceof WeightedSetDataType) { - WeightedSetDataType wType = (WeightedSetDataType) type; - visit(wType.getNestedType(), owningDocument); - } else if (type instanceof CollectionDataType) { - CollectionDataType cType = (CollectionDataType) type; - visit(cType.getNestedType(), owningDocument); - } else if (type instanceof AnnotationReferenceDataType) { - //do nothing - } else if (type instanceof PrimitiveDataType) { - //do nothing - } else if (type instanceof TensorDataType) { - //do nothing - } else if (type instanceof NewDocumentReferenceDataType) { - //do nothing - } else { - deployLogger.logApplicationPackage(Level.WARNING, "Unknown type : " + type); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/schema/TemporarySDTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/TemporarySDTypeResolver.java deleted file mode 100644 index b1ce6f5eb4f..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/TemporarySDTypeResolver.java +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.schema.document.SDDocumentType; -import com.yahoo.schema.document.TemporarySDDocumentType; - -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.logging.Level; - -/** - * @author arnej - */ -public class TemporarySDTypeResolver { - - private final DeployLogger deployLogger; - private final Collection<Schema> toProcess; - private final List<SDDocumentType> docTypes = new LinkedList<>(); - - public TemporarySDTypeResolver(Collection<Schema> schemas, DeployLogger deployLogger) { - this.deployLogger = deployLogger; - this.toProcess = schemas; - } - - private SDDocumentType findDocType(String name) { - assert(name != null); - for (var doc : docTypes) { - if (doc.getName().equals(name)) { - return doc; - } - } - deployLogger.logApplicationPackage(Level.WARNING, "No document type in application matching name: "+name); - return null; - } - - public void process() { - docTypes.add(SDDocumentType.VESPA_DOCUMENT); - for (Schema schema : toProcess) { - if (schema.hasDocument()) { - docTypes.add(schema.getDocument()); - } - } - // first, fix inheritance - for (SDDocumentType doc : docTypes) { - for (SDDocumentType inherited : doc.getInheritedTypes()) { - if (inherited instanceof TemporarySDDocumentType) { - var actual = findDocType(inherited.getName()); - if (actual != null) { - doc.inherit(actual); - } else { - deployLogger.logApplicationPackage(Level.WARNING, "Unresolved inherit '"+inherited.getName() +"' for document "+doc.getName()); - } - } - } - } - // next, check owned types (structs only?) - for (SDDocumentType doc : docTypes) { - for (SDDocumentType owned : doc.getTypes()) { - if (owned instanceof TemporarySDDocumentType) { - deployLogger.logApplicationPackage(Level.WARNING, "Schema '"+doc.getName()+"' owned type '"+owned.getName()+"' is temporary, should not happen"); - continue; - } - for (SDDocumentType inherited : owned.getInheritedTypes()) { - if (inherited instanceof TemporarySDDocumentType) { - var actual = doc.getType(inherited.getName()); - if (actual != null) { - owned.inherit(actual); - } else { - deployLogger.logApplicationPackage(Level.WARNING, "Unresolved inherit '"+inherited.getName() +"' for type '"+owned.getName()+"' in document "+doc.getName()); - } - } - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java index 162efbb25b4..eb336e1fc72 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java @@ -15,20 +15,22 @@ import java.util.Set; */ public class Juniperrc extends Derived implements JuniperrcConfig.Producer { - // List of all fields that should be bolded. - private Set<String> boldingFields = new java.util.LinkedHashSet<>(); + private static final int Mb = 1024 * 1024; + + /** List of all fields that should be bolded. */ + private final Set<String> boldingFields = new java.util.LinkedHashSet<>(); /** - * Constructs a new juniper rc instance for a given search object. This will derive the configuration automatically, + * Constructs a new juniper rc instance for a given search object. + * This will derive the configuration automatically, * so there is no need to call {@link #derive(Schema)}. * - * @param schema The search model to use for deriving. + * @param schema the search model to use for deriving */ public Juniperrc(Schema schema) { derive(schema); } - // Inherit doc from Derived. @Override protected void derive(Schema schema) { super.derive(schema); @@ -39,11 +41,8 @@ public class Juniperrc extends Derived implements JuniperrcConfig.Producer { } } - // Inherit doc from Derived. @Override - protected String getDerivedName() { - return "juniperrc"; - } + protected String getDerivedName() { return "juniperrc"; } @Override public void getConfig(JuniperrcConfig.Builder builder) { @@ -52,11 +51,12 @@ public class Juniperrc extends Derived implements JuniperrcConfig.Producer { for (String name : boldingFields) { builder.override(new JuniperrcConfig.Override.Builder() .fieldname(name) - .length(65536) + .length(64*Mb) .max_matches(1) .min_length(8192) - .surround_max(65536)); + .surround_max(64*Mb)); } } } + } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index 66389a030af..11af3b0bf78 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -154,7 +154,6 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final OptionalDouble postFilterThreshold; private final OptionalDouble approximateThreshold; private final double rankScoreDropLimit; - private final boolean mapBackRankingExpressionFeatures; /** * The rank type definitions used to derive settings for the native rank features @@ -193,7 +192,6 @@ public class RawRankProfile implements RankProfilesConfig.Producer { approximateThreshold = compiled.getApproximateThreshold(); keepRankCount = compiled.getKeepRankCount(); rankScoreDropLimit = compiled.getRankScoreDropLimit(); - mapBackRankingExpressionFeatures = deployProperties.featureFlags().avoidRenamingSummaryFeatures(); ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); rankProperties = new ArrayList<>(compiled.getRankProperties()); @@ -274,10 +272,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { ReferenceNode backendReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); - if (mapBackRankingExpressionFeatures) { - // tell backend to map back to the name the user expects: - featureRenames.put(backendReferenceNode.toString(), referenceNode.toString()); - } + // tell backend to map back to the name the user expects: + featureRenames.put(backendReferenceNode.toString(), referenceNode.toString()); functionFeatures.put(referenceNode.getName(), backendReferenceNode); i.remove(); // Will add the expanded one in next block } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java b/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java index 3bab808beff..d08cc472f82 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java @@ -33,10 +33,6 @@ public class SearchOrderer { * @return a new list containing the same search instances in the right order */ public List<Schema> order(List<Schema> unordered) { - // Description above state that the original order should be preserved, except for the dependency constraint. - // Yet we botch that guarantee by sorting the list... - unordered.sort(Comparator.comparing(Schema::getName)); - // No, this is not a fast algorithm... indexOnDocumentName(unordered); List<Schema> ordered = new ArrayList<>(unordered.size()); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java index df9174a12ed..35bb147ccf5 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java @@ -42,6 +42,8 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { if (summaryField.getTransform()== SummaryTransform.NONE) continue; if (summaryField.getTransform()==SummaryTransform.ATTRIBUTE || + (summaryField.getTransform()==SummaryTransform.ATTRIBUTECOMBINER && summaryField.hasExplicitSingleSource()) || + summaryField.getTransform()==SummaryTransform.COPY || summaryField.getTransform()==SummaryTransform.DISTANCE || summaryField.getTransform()==SummaryTransform.GEOPOS || summaryField.getTransform()==SummaryTransform.POSITIONS || @@ -57,9 +59,10 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { // This works, but is suboptimal. We could consolidate to a minimal set and // use the right value from the minimal set as the third parameter here, // and add "override" commands to multiple static values + boolean useFieldNameAsArgument = summaryField.getTransform().isDynamic() || summaryField.getTransform() == SummaryTransform.TEXTEXTRACTOR; resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), summaryField.getTransform(), - summaryField.getName())); + useFieldNameAsArgument ? summaryField.getName() : "")); } } } @@ -99,20 +102,8 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { for (FieldResultTransform frt : resultTransforms.values()) { SummarymapConfig.Override.Builder oB = new SummarymapConfig.Override.Builder() .field(frt.getFieldName()) - .command(getCommand(frt.getTransform())); - if (frt.getTransform().isDynamic() || - frt.getTransform().equals(SummaryTransform.ATTRIBUTE) || - frt.getTransform().equals(SummaryTransform.DISTANCE) || - frt.getTransform().equals(SummaryTransform.GEOPOS) || - frt.getTransform().equals(SummaryTransform.POSITIONS) || - frt.getTransform().equals(SummaryTransform.TEXTEXTRACTOR) || - frt.getTransform().equals(SummaryTransform.MATCHED_ELEMENTS_FILTER) || - frt.getTransform().equals(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER)) - { - oB.arguments(frt.getArgument()); - } else { - oB.arguments(""); - } + .command(getCommand(frt.getTransform())) + .arguments(frt.getArgument()); builder.override(oB); } } diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java index d19b62ce555..919a6023151 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java @@ -167,16 +167,11 @@ public class SDDocumentType implements Cloneable { public DataTypeName getDocumentName() { return docType.getDataTypeName(); } public DocumentType getDocumentType() { return docType; } - public void inherit(DataTypeName name) { - inherit(new TemporarySDDocumentType(name)); - } - public void inherit(SDDocumentType type) { if (type == null) return; if (type.getName().equals(this.getName())) throw new IllegalArgumentException("Document type '" + getName() + "' cannot inherit itself"); - if ( ! inheritedTypes.containsKey(type.getDocumentName()) || - (inheritedTypes.get(type.getDocumentName()) instanceof TemporarySDDocumentType)) { + if (! inheritedTypes.containsKey(type.getDocumentName())) { inheritedTypes.put(type.getDocumentName(), type); } } diff --git a/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java deleted file mode 100644 index 3194a14a143..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.document; - -import com.yahoo.document.DataTypeName; - -/** - * @author baldersheim - */ -public class TemporarySDDocumentType extends SDDocumentType { - public TemporarySDDocumentType(DataTypeName name) { - super(name); - } -} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index 3008dac4f29..443dfb1602d 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import java.util.Locale; +import java.util.Map; /** * Helper for converting ParsedField etc to SDField with settings @@ -29,9 +30,11 @@ import java.util.Locale; public class ConvertParsedFields { private final TypeResolver context; + private final Map<String, SDDocumentType> structProxies; - ConvertParsedFields(TypeResolver context) { + ConvertParsedFields(TypeResolver context, Map<String, SDDocumentType> structProxies) { this.context = context; + this.structProxies = structProxies; } static void convertMatchSettings(SDField field, ParsedMatchSettings parsed) { @@ -309,10 +312,11 @@ public class ConvertParsedFields { structProxy.setFieldId(field, parsedField.idOverride()); } } - for (String inherit : parsed.getInherited()) { - structProxy.inherit(new DataTypeName(inherit)); + for (var inherit: parsed.getResolvedInherits()) { + structProxy.inherit(structProxies.get(inherit.getFullName())); } structProxy.setStruct(context.resolveStruct(parsed)); + structProxies.put(parsed.getFullName(), structProxy); return structProxy; } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java index f3289621ce1..0abcc9e890a 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -80,6 +80,7 @@ public class ConvertParsedSchemas { } private final Map<String, SDDocumentType> convertedDocuments = new LinkedHashMap<>(); + private final Map<String, SDDocumentType> convertedStructs = new LinkedHashMap<>(); public List<Schema> convertToSchemas() { typeConverter.convert(false); @@ -187,7 +188,7 @@ public class ConvertParsedSchemas { } parsed.getRawAsBase64().ifPresent(value -> schema.enableRawAsBase64(value)); var typeContext = typeConverter.makeContext(parsed.getDocument()); - var fieldConverter = new ConvertParsedFields(typeContext); + var fieldConverter = new ConvertParsedFields(typeContext, convertedStructs); convertDocument(schema, parsed.getDocument(), fieldConverter); for (var field : parsed.getFields()) { fieldConverter.convertExtraField(schema, field); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java index abe14b3689f..02d10bcb487 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java @@ -27,8 +27,13 @@ public class ParsedStruct extends ParsedBlock { List<ParsedField> getFields() { return List.copyOf(fields.values()); } List<String> getInherited() { return List.copyOf(inherited); } - ParsedDocument getOwnerDoc() { return ownedBy; } - String getOwnerName() { return ownedBy.name(); } + ParsedDocument getOwnerDoc() { + verifyThat(ownedBy != null, "not owned by any document"); + return ownedBy; + } + String getOwnerName() { return getOwnerDoc().name(); } + String getFullName() { return name() + " @ " + getOwnerName(); } + List<ParsedStruct> getResolvedInherits() { assert(inherited.size() == resolvedInherits.size()); return List.copyOf(resolvedInherits); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java index 4fb45c3c68f..4aa8f6f0e37 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java @@ -37,6 +37,7 @@ public class SummaryConsistency extends Processor { assertConsistency(summaryField, schema, validate); makeAttributeTransformIfAppropriate(summaryField, schema); makeAttributeCombinerTransformIfAppropriate(summaryField, schema); + makeCopyTransformIfAppropriate(summaryField, schema); } } } @@ -70,14 +71,27 @@ public class SummaryConsistency extends Processor { /** If the source is a complex field with only struct field attributes then make this use the attribute combiner transform */ private void makeAttributeCombinerTransformIfAppropriate(SummaryField summaryField, Schema schema) { if (summaryField.getTransform() == SummaryTransform.NONE) { - String source_field_name = summaryField.getSingleSource(); - ImmutableSDField source = schema.getField(source_field_name); + String sourceFieldName = summaryField.getSingleSource(); + ImmutableSDField source = schema.getField(sourceFieldName); if (source != null && isComplexFieldWithOnlyStructFieldAttributes(source)) { summaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); } } } + /* + * This function must be called after makeAttributeCombinerTransformIfAppropriate(). + */ + private void makeCopyTransformIfAppropriate(SummaryField summaryField, Schema schema) { + if (summaryField.getTransform() == SummaryTransform.NONE) { + String sourceFieldName = summaryField.getSingleSource(); + ImmutableSDField source = schema.getField(sourceFieldName); + if (source != null && source.usesStructOrMap() && summaryField.hasExplicitSingleSource()) { + summaryField.setTransform(SummaryTransform.COPY); + } + } + } + private void assertConsistentTypes(SummaryField existing, SummaryField seen) { if (existing.getDataType() instanceof WeightedSetDataType && seen.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)existing.getDataType()).getNestedType().equals(((WeightedSetDataType)seen.getDataType()).getNestedType())) diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index ed6668f0d0d..58044163885 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -263,6 +263,22 @@ public class SummaryField extends Field implements Cloneable, TypedKey { } } + /** + * Returns true if the summary field uses an explicit source, i.e. + * a field with different name that is not a nested field. + */ + public boolean hasExplicitSingleSource() { + String fieldName = getName(); + String sourceName = getSingleSource(); + if (fieldName.equals(sourceName)) { + return false; + } + if (sourceName.contains(".")) { + return false; + } + return true; + } + public VsmCommand getVsmCommand() { return vsmCommand; } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index c50766a2585..a1ebc9e4bec 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -22,7 +22,8 @@ public enum SummaryTransform { GEOPOS("geopos"), ATTRIBUTECOMBINER("attributecombiner"), MATCHED_ELEMENTS_FILTER("matchedelementsfilter"), - MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"); + MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"), + COPY("copy"); private final String name; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 52a51a2c323..47dab37cc14 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -28,7 +28,6 @@ public final class ApplicationContainer extends Container implements private final boolean isHostedVespa; private final boolean enableServerOcspStapling; - private final boolean useQrserverServiceName; public ApplicationContainer(AbstractConfigProducer<?> parent, String name, int index, DeployState deployState) { this(parent, name, false, index, deployState); @@ -38,7 +37,6 @@ public final class ApplicationContainer extends Container implements super(parent, name, retired, index, deployState); this.isHostedVespa = deployState.isHosted(); this.enableServerOcspStapling = deployState.featureFlags().enableServerOcspStapling(); - this.useQrserverServiceName = false; addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder")); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); @@ -59,13 +57,6 @@ public final class ApplicationContainer extends Container implements @Override protected ContainerServiceType myServiceType() { - if (parent instanceof ContainerCluster) { - ContainerCluster<?> cluster = (ContainerCluster<?>)parent; - // TODO: The 'qrserver' name is retained for legacy reasons (e.g. system tests and log parsing). - if (useQrserverServiceName && cluster.getSearch() != null && cluster.getDocproc() == null && cluster.getDocumentApi() == null) { - return ContainerServiceType.QRSERVER; - } - } return ContainerServiceType.CONTAINER; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 4755f674f69..47633fc5191 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -227,8 +227,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> public void addApplicationStatusHandler() { Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>( - new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings( - APPLICATION_STATUS_HANDLER_CLASS, null), null)); + new ComponentModel(BundleInstantiationSpecification.getFromStrings( + APPLICATION_STATUS_HANDLER_CLASS, null, null), null)); // null bundle, as the handler is in container-disc statusHandler.addServerBindings(SystemBindingPattern.fromHttpPath("/ApplicationStatus")); addComponent(statusHandler); } @@ -302,7 +302,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> // Cannot use the class object for ProcessingHandler, because its superclass is not accessible ProcessingHandler<?> processingHandler = new ProcessingHandler<>( processingChains, - "com.yahoo.processing.handler.ProcessingHandler"); + "com.yahoo.processing.handler.ProcessingHandler", + null); for (BindingPattern binding: serverBindings) processingHandler.addServerBindings(binding); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java index 71524d7e157..3f68a0a2709 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java @@ -20,9 +20,16 @@ public class ProcessingHandler<CHAINS extends Chains<?>> protected final CHAINS chains; public ProcessingHandler(CHAINS chains, String handlerClass) { - super(new ComponentModel(BundleInstantiationSpecification.getInternalProcessingSpecificationFromStrings(handlerClass, null), null)); - this.chains = chains; + this(chains, BundleInstantiationSpecification.getInternalProcessingSpecificationFromStrings(handlerClass, null)); + } + public ProcessingHandler(CHAINS chains, String handlerClass, String bundle) { + this(chains, BundleInstantiationSpecification.getFromStrings(handlerClass, null, bundle)); + } + + private ProcessingHandler(CHAINS chains, BundleInstantiationSpecification spec) { + super(new ComponentModel(spec, null)); + this.chains = chains; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java index 4b9897d0950..31ba195775b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java @@ -3,7 +3,10 @@ package com.yahoo.vespa.model.container.docproc; import com.yahoo.component.ComponentId; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.jdisc.config.SessionConfig; +import com.yahoo.docproc.jdisc.observability.DocprocsStatusExtension; +import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Component; @@ -25,6 +28,11 @@ public class DocprocChains extends Chains<DocprocChain> { super(parent, subId); docprocHandler = new ProcessingHandler<>(this, "com.yahoo.docproc.jdisc.DocumentProcessingHandler"); addComponent(docprocHandler); + addComponent( + new SimpleComponent( + new ComponentModel( + BundleInstantiationSpecification.getInternalProcessingSpecificationFromStrings( + DocprocsStatusExtension.class.getName(), null), null))); if (! (getParent() instanceof ApplicationContainerCluster)) { // All application containers already have a DocumentTypeManager, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index f27e0b24c82..a42d4a665ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.container.http.xml; import com.yahoo.component.ComponentSpecification; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -19,7 +18,6 @@ import com.yahoo.vespa.model.container.http.FilterBinding; import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; import org.w3c.dom.Element; - import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -36,7 +34,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> static final List<String> VALID_FILTER_CHAIN_TAG_NAMES = List.of(REQUEST_CHAIN_TAG_NAME, RESPONSE_CHAIN_TAG_NAME); @Override - protected Http doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element spec) { + protected Http doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element spec) { FilterChains filterChains; List<FilterBinding> bindings = new ArrayList<>(); AccessControl accessControl = null; @@ -74,16 +72,18 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> getContainerCluster(ancestor).ifPresent(builder::setHandlers); + // TODO: Remove in Vespa 9 XmlHelper.getOptionalAttribute(accessControlElem, "read").ifPresent( readAttr -> deployState.getDeployLogger() .logApplicationPackage(Level.WARNING, "The 'read' attribute of the 'access-control' element has no effect and is deprecated. " + - "Please remove the attribute from services.xml")); + "Please remove the attribute from services.xml, support will be removed in Vespa 9")); + // TODO: Remove in Vespa 9 XmlHelper.getOptionalAttribute(accessControlElem, "write").ifPresent( writeAttr -> deployState.getDeployLogger() .logApplicationPackage(Level.WARNING, "The 'write' attribute of the 'access-control' element has no effect and is deprecated. " + - "Please remove the attribute from services.xml")); + "Please remove the attribute from services.xml, support will be removed in Vespa 9")); AccessControl.ClientAuthentication clientAuth = XmlHelper.getOptionalAttribute(accessControlElem, "tls-handshake-client-auth") @@ -129,8 +129,8 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> return tenantDomain != null ? tenantDomain : explicitDomain; } - private static Optional<ApplicationContainerCluster> getContainerCluster(AbstractConfigProducer configProducer) { - AbstractConfigProducer currentProducer = configProducer; + private static Optional<ApplicationContainerCluster> getContainerCluster(AbstractConfigProducer<?> configProducer) { + AbstractConfigProducer<?> currentProducer = configProducer; while (! ApplicationContainerCluster.class.isAssignableFrom(currentProducer.getClass())) { currentProducer = currentProducer.getParent(); if (currentProducer == null) @@ -164,7 +164,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> } } - static int readPort(ModelElement spec, boolean isHosted, DeployLogger logger) { + static int readPort(ModelElement spec, boolean isHosted) { Integer port = spec.integerAttribute("port"); if (port == null) return Defaults.getDefaults().vespaWebServicePort(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java index c6fae15d06a..69480ee0703 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java @@ -30,7 +30,7 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil @Override protected ConnectorFactory doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element serverSpec) { String name = XmlHelper.getIdString(serverSpec); - int port = HttpBuilder.readPort(new ModelElement(serverSpec), deployState.isHosted(), deployState.getDeployLogger()); + int port = HttpBuilder.readPort(new ModelElement(serverSpec), deployState.isHosted()); ConnectorFactory.Builder builder = new ConnectorFactory.Builder(name, port); XmlHelper.getOptionalAttribute(serverSpec, "default-request-chain") .map(ComponentId::new) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 97c3f85dced..58151063956 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -5,6 +5,7 @@ import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.search.handler.observability.SearchStatusExtension; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfilesConfig; @@ -57,6 +58,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> owningCluster.addComponent(Component.fromClassAndBundle(QUERY_PROFILE_REGISTRY_CLASS, searchAndDocprocBundle)); owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class.getName(), searchAndDocprocBundle)); + owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class.getName(), searchAndDocprocBundle)); } public void connectSearchClusters(Map<String, SearchCluster> searchClusters) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index f971aa97e6a..1aab98a115f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -187,6 +187,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addEmbedderComponents(deployState, cluster, spec); addModelEvaluation(spec, cluster, context); + addVespaBundles(cluster); addModelEvaluationBundles(cluster); addProcessing(deployState, spec, cluster); @@ -583,6 +584,17 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return (child != null) ? Integer.parseInt(child.getTextContent()) : defaultValue; } + private void addVespaBundles(ApplicationContainerCluster cluster) { + // Skip model-evaluation bundles for node-admin + if (cluster.id().value().equals("node-admin")) return; + + Set<String> bundles = Set.of( + "container-search-and-docproc", "container-search-gui", "docprocs", + "linguistics-components", "vespaclient-container-plugin"); + bundles.forEach(b -> cluster.addPlatformBundle(PlatformBundles.absoluteBundlePath(b))); + addModelEvaluationBundles(cluster); + } + protected void addModelEvaluationBundles(ApplicationContainerCluster cluster) { /* These bundles are added to all application container clusters, even if they haven't * declared 'model-evaluation' in services.xml, because there are many public API packages diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 379115ab3ca..ff39aa0903c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -67,6 +67,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; private final double defaultFeedConcurrency; + private final double defaultFeedNiceness; private final boolean forwardIssuesToQrs; private final int defaultMaxCompactBuffers; @@ -213,6 +214,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.fractionOfMemoryReserved = fractionOfMemoryReserved; this.feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); this.defaultFeedConcurrency = featureFlags.feedConcurrency(); + this.defaultFeedNiceness = featureFlags.feedNiceness(); this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors(); this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers(); } @@ -399,6 +401,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } else { builder.feeding.concurrency(defaultFeedConcurrency); } + builder.feeding.niceness(defaultFeedNiceness); builder.flush.memory.diskbloatfactor(DEFAULT_DISK_BLOAT); builder.flush.memory.each.diskbloatfactor(DEFAULT_DISK_BLOAT); builder.summary.log.chunk.compression.level(DEFAULT_DOC_STORE_COMPRESSION_LEVEL); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java index 2215a0a8df9..b4c48b3204b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -71,13 +70,6 @@ public class Distributor extends ContentNode implements StorDistributormanagerCo } @Override - public void getConfig(StorCommunicationmanagerConfig.Builder builder) { - super.getConfig(builder); - // Single distributor needs help to encode the messages. - builder.mbus.dispatch_on_encode(true); - } - - @Override public void getConfig(StorDistributormanagerConfig.Builder builder) { builder.num_distributor_stripes(tuneNumDistributorStripes()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java index 1bdd281d38e..af103d4d5b9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java @@ -6,7 +6,6 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorBucketmoverConfig; -import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.application.validation.RestartConfigs; @@ -94,10 +93,4 @@ public class StorageNode extends ContentNode implements StorServerConfig.Produce cluster.getConfig(builder); } - @Override - public void getConfig(StorCommunicationmanagerConfig.Builder builder) { - super.getConfig(builder); - builder.mbus.dispatch_on_encode(false); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 9efe61d134f..2c83e87df97 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -215,8 +215,6 @@ public class IndexedSearchCluster extends SearchCluster for (DocumentDatabase db : documentDbs) { DocumentdbInfoConfig.Documentdb.Builder docDb = new DocumentdbInfoConfig.Documentdb.Builder(); docDb.name(db.getName()); - convertSummaryConfig(db, db, docDb); - addRankProfilesConfig(db.getSchemaName(), docDb); builder.documentdb(docDb); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index f033399c787..0fe8a1b837a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -4,12 +4,9 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.schema.derived.SchemaInfo; -import com.yahoo.schema.derived.SummaryMap; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -58,71 +55,6 @@ public abstract class SearchCluster extends AbstractConfigProducer<SearchCluster */ public abstract void deriveFromSchemas(DeployState deployState); - /** - * Converts summary and summary map config to the appropriate information in documentdb - * - * @param summaryConfigProducer the summary config - * @param summarymapConfigProducer the summary map config, or null if none is available - * @param docDb the target document dm config - */ - protected void convertSummaryConfig(SummaryConfig.Producer summaryConfigProducer, - SummarymapConfig.Producer summarymapConfigProducer, - DocumentdbInfoConfig.Documentdb.Builder docDb) { - - SummaryConfig.Builder summaryConfigBuilder = new SummaryConfig.Builder(); - summaryConfigProducer.getConfig(summaryConfigBuilder); - SummaryConfig summaryConfig = summaryConfigBuilder.build(); - - SummarymapConfig summarymapConfig = null; - if (summarymapConfigProducer != null) { - SummarymapConfig.Builder summarymapConfigBuilder = new SummarymapConfig.Builder(); - summarymapConfigProducer.getConfig(summarymapConfigBuilder); - summarymapConfig = summarymapConfigBuilder.build(); - } - - for (SummaryConfig.Classes sclass : summaryConfig.classes()) { - DocumentdbInfoConfig.Documentdb.Summaryclass.Builder sumClassBuilder = new DocumentdbInfoConfig.Documentdb.Summaryclass.Builder(); - sumClassBuilder. - id(sclass.id()). - name(sclass.name()); - for (SummaryConfig.Classes.Fields field : sclass.fields()) { - DocumentdbInfoConfig.Documentdb.Summaryclass.Fields.Builder fieldsBuilder = new DocumentdbInfoConfig.Documentdb.Summaryclass.Fields.Builder(); - fieldsBuilder.name(field.name()) - .type(field.type()) - .dynamic(isDynamic(field.name(), summarymapConfig)); - sumClassBuilder.fields(fieldsBuilder); - } - docDb.summaryclass(sumClassBuilder); - } - } - - /** Returns whether the given field is a dynamic summary field. */ - private boolean isDynamic(String fieldName, SummarymapConfig summarymapConfig) { - if (summarymapConfig == null) return false; // not know for streaming, but also not used - - for (SummarymapConfig.Override override : summarymapConfig.override()) { - if ( ! fieldName.equals(override.field())) continue; - if (SummaryMap.isDynamicCommand(override.command())) return true; - } - return false; - } - - protected void addRankProfilesConfig(String schemaName, DocumentdbInfoConfig.Documentdb.Builder docDbBuilder) { - for (SchemaInfo.RankProfileInfo rankProfile : schemas().get(schemaName).rankProfiles().values()) { - var rankProfileConfig = new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder(); - rankProfileConfig.name(rankProfile.name()); - rankProfileConfig.hasSummaryFeatures(rankProfile.hasSummaryFeatures()); - rankProfileConfig.hasRankFeatures(rankProfile.hasRankFeatures()); - for (var input : rankProfile.inputs().entrySet()) { - var inputConfig = new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder(); - inputConfig.name(input.getKey().toString()); - inputConfig.type(input.getValue().type().toString()); - rankProfileConfig.input(inputConfig); - } - docDbBuilder.rankprofile(rankProfileConfig); - } - } - /** Returns a list of the document type names used in this search cluster */ public List<String> getDocumentNames() { return schemas.values() diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 79adfb2ea13..d12635b76ec 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -65,9 +65,6 @@ public class StreamingSearchCluster extends SearchCluster implements public void getConfig(DocumentdbInfoConfig.Builder builder) { DocumentdbInfoConfig.Documentdb.Builder docDb = new DocumentdbInfoConfig.Documentdb.Builder(); docDb.name(derivedConfig.getSchema().getName()); - SummaryConfig.Producer prod = derivedConfig.getSummaries(); - convertSummaryConfig(prod, null, docDb); - addRankProfilesConfig(derivedConfig.getSchema().getName(), docDb); builder.documentdb(docDb); } diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg index eaa7e2c5131..a4b37113ef1 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -39,48 +39,48 @@ doctype[0].structtype[1].field[0].type 10007 doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 -doctype[1].name "ad" +doctype[1].name "campaign" doctype[1].idx 10015 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" -doctype[1].fieldsets{[document]}.fields[1] "person_ref" -doctype[1].importedfield[0].name "my_cool_field" -doctype[1].importedfield[1].name "my_swag_field" -doctype[1].importedfield[2].name "my_name" -doctype[1].documentref[0].idx 10017 -doctype[1].documentref[0].targettype 10018 -doctype[1].documentref[1].idx 10019 -doctype[1].documentref[1].targettype 10020 +doctype[1].fieldsets{[document]}.fields[0] "cool_field" +doctype[1].fieldsets{[document]}.fields[1] "swag_field" doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "ad.header" -doctype[1].structtype[0].field[0].name "campaign_ref" -doctype[1].structtype[0].field[0].internalid 23963250 -doctype[1].structtype[0].field[0].type 10017 -doctype[1].structtype[0].field[1].name "person_ref" -doctype[1].structtype[0].field[1].internalid 100779805 -doctype[1].structtype[0].field[1].type 10019 -doctype[2].name "campaign" -doctype[2].idx 10018 +doctype[1].structtype[0].name "campaign.header" +doctype[1].structtype[0].field[0].name "cool_field" +doctype[1].structtype[0].field[0].internalid 1588702436 +doctype[1].structtype[0].field[0].type 10012 +doctype[1].structtype[0].field[1].name "swag_field" +doctype[1].structtype[0].field[1].internalid 1691224741 +doctype[1].structtype[0].field[1].type 10008 +doctype[2].name "person" +doctype[2].idx 10017 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10021 -doctype[2].fieldsets{[document]}.fields[0] "cool_field" -doctype[2].fieldsets{[document]}.fields[1] "swag_field" -doctype[2].structtype[0].idx 10021 -doctype[2].structtype[0].name "campaign.header" -doctype[2].structtype[0].field[0].name "cool_field" -doctype[2].structtype[0].field[0].internalid 1588702436 +doctype[2].contentstruct 10018 +doctype[2].fieldsets{[document]}.fields[0] "name" +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "person.header" +doctype[2].structtype[0].field[0].name "name" +doctype[2].structtype[0].field[0].internalid 1160796772 doctype[2].structtype[0].field[0].type 10012 -doctype[2].structtype[0].field[1].name "swag_field" -doctype[2].structtype[0].field[1].internalid 1691224741 -doctype[2].structtype[0].field[1].type 10008 -doctype[3].name "person" -doctype[3].idx 10020 +doctype[3].name "ad" +doctype[3].idx 10019 doctype[3].inherits[0].idx 10000 -doctype[3].contentstruct 10022 -doctype[3].fieldsets{[document]}.fields[0] "name" -doctype[3].structtype[0].idx 10022 -doctype[3].structtype[0].name "person.header" -doctype[3].structtype[0].field[0].name "name" -doctype[3].structtype[0].field[0].internalid 1160796772 -doctype[3].structtype[0].field[0].type 10012 +doctype[3].contentstruct 10020 +doctype[3].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[3].fieldsets{[document]}.fields[1] "person_ref" +doctype[3].importedfield[0].name "my_cool_field" +doctype[3].importedfield[1].name "my_swag_field" +doctype[3].importedfield[2].name "my_name" +doctype[3].documentref[0].idx 10021 +doctype[3].documentref[0].targettype 10015 +doctype[3].documentref[1].idx 10022 +doctype[3].documentref[1].targettype 10017 +doctype[3].structtype[0].idx 10020 +doctype[3].structtype[0].name "ad.header" +doctype[3].structtype[0].field[0].name "campaign_ref" +doctype[3].structtype[0].field[0].internalid 23963250 +doctype[3].structtype[0].field[0].type 10021 +doctype[3].structtype[0].field[1].name "person_ref" +doctype[3].structtype[0].field[1].internalid 100779805 +doctype[3].structtype[0].field[1].type 10022 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg index 48348348989..f6d4269abcd 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg @@ -39,33 +39,33 @@ doctype[0].structtype[1].field[0].type 10007 doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 -doctype[1].name "ad" +doctype[1].name "campaign" doctype[1].idx 10015 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" -doctype[1].fieldsets{[document]}.fields[1] "person_ref" -doctype[1].documentref[0].idx 10017 -doctype[1].documentref[0].targettype 10018 -doctype[1].documentref[1].idx 10019 -doctype[1].documentref[1].targettype 10020 doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "ad.header" -doctype[1].structtype[0].field[0].name "campaign_ref" -doctype[1].structtype[0].field[0].internalid 23963250 -doctype[1].structtype[0].field[0].type 10017 -doctype[1].structtype[0].field[1].name "person_ref" -doctype[1].structtype[0].field[1].internalid 100779805 -doctype[1].structtype[0].field[1].type 10019 -doctype[2].name "campaign" -doctype[2].idx 10018 +doctype[1].structtype[0].name "campaign.header" +doctype[2].name "person" +doctype[2].idx 10017 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10021 -doctype[2].structtype[0].idx 10021 -doctype[2].structtype[0].name "campaign.header" -doctype[3].name "person" -doctype[3].idx 10020 +doctype[2].contentstruct 10018 +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "person.header" +doctype[3].name "ad" +doctype[3].idx 10019 doctype[3].inherits[0].idx 10000 -doctype[3].contentstruct 10022 -doctype[3].structtype[0].idx 10022 -doctype[3].structtype[0].name "person.header" +doctype[3].contentstruct 10020 +doctype[3].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[3].fieldsets{[document]}.fields[1] "person_ref" +doctype[3].documentref[0].idx 10021 +doctype[3].documentref[0].targettype 10015 +doctype[3].documentref[1].idx 10022 +doctype[3].documentref[1].targettype 10017 +doctype[3].structtype[0].idx 10020 +doctype[3].structtype[0].name "ad.header" +doctype[3].structtype[0].field[0].name "campaign_ref" +doctype[3].structtype[0].field[0].internalid 23963250 +doctype[3].structtype[0].field[0].type 10021 +doctype[3].structtype[0].field[1].name "person_ref" +doctype[3].structtype[0].field[1].internalid 100779805 +doctype[3].structtype[0].field[1].type 10022 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg index ef1bb4c5ad4..31f514b16cd 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg @@ -39,25 +39,25 @@ doctype[0].structtype[1].field[0].type 10007 doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 -doctype[1].name "ad" +doctype[1].name "campaign" doctype[1].idx 10015 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" -doctype[1].fieldsets{[document]}.fields[1] "other_campaign_ref" -doctype[1].documentref[0].idx 10017 -doctype[1].documentref[0].targettype 10018 doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "ad.header" -doctype[1].structtype[0].field[0].name "campaign_ref" -doctype[1].structtype[0].field[0].internalid 23963250 -doctype[1].structtype[0].field[0].type 10017 -doctype[1].structtype[0].field[1].name "other_campaign_ref" -doctype[1].structtype[0].field[1].internalid 874751172 -doctype[1].structtype[0].field[1].type 10017 -doctype[2].name "campaign" -doctype[2].idx 10018 +doctype[1].structtype[0].name "campaign.header" +doctype[2].name "ad" +doctype[2].idx 10017 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10019 -doctype[2].structtype[0].idx 10019 -doctype[2].structtype[0].name "campaign.header" +doctype[2].contentstruct 10018 +doctype[2].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[2].fieldsets{[document]}.fields[1] "other_campaign_ref" +doctype[2].documentref[0].idx 10019 +doctype[2].documentref[0].targettype 10015 +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "ad.header" +doctype[2].structtype[0].field[0].name "campaign_ref" +doctype[2].structtype[0].field[0].internalid 23963250 +doctype[2].structtype[0].field[0].type 10019 +doctype[2].structtype[0].field[1].name "other_campaign_ref" +doctype[2].structtype[0].field[1].internalid 874751172 +doctype[2].structtype[0].field[1].type 10019 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg index 1b5817e6f39..1582c6572da 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -43,56 +43,56 @@ doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 doctype[0].structtype[1].internalid 1381038251 -doctype[1].name "ad" +doctype[1].name "campaign" doctype[1].idx 10015 -doctype[1].internalid 2987301 +doctype[1].internalid -1318255918 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" -doctype[1].fieldsets{[document]}.fields[1] "person_ref" -doctype[1].importedfield[0].name "my_cool_field" -doctype[1].importedfield[1].name "my_swag_field" -doctype[1].importedfield[2].name "my_name" -doctype[1].documentref[0].idx 10017 -doctype[1].documentref[0].targettype 10018 -doctype[1].documentref[0].internalid 595216861 -doctype[1].documentref[1].idx 10019 -doctype[1].documentref[1].targettype 10020 -doctype[1].documentref[1].internalid 542332920 +doctype[1].fieldsets{[document]}.fields[0] "cool_field" +doctype[1].fieldsets{[document]}.fields[1] "swag_field" doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "ad.header" -doctype[1].structtype[0].field[0].name "campaign_ref" -doctype[1].structtype[0].field[0].internalid 23963250 -doctype[1].structtype[0].field[0].type 10017 -doctype[1].structtype[0].field[1].name "person_ref" -doctype[1].structtype[0].field[1].internalid 100779805 -doctype[1].structtype[0].field[1].type 10019 -doctype[1].structtype[0].internalid 959075962 -doctype[2].name "campaign" -doctype[2].idx 10018 -doctype[2].internalid -1318255918 +doctype[1].structtype[0].name "campaign.header" +doctype[1].structtype[0].field[0].name "cool_field" +doctype[1].structtype[0].field[0].internalid 1588702436 +doctype[1].structtype[0].field[0].type 10012 +doctype[1].structtype[0].field[1].name "swag_field" +doctype[1].structtype[0].field[1].internalid 1691224741 +doctype[1].structtype[0].field[1].type 10008 +doctype[1].structtype[0].internalid -2041471955 +doctype[2].name "person" +doctype[2].idx 10017 +doctype[2].internalid 443162583 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10021 -doctype[2].fieldsets{[document]}.fields[0] "cool_field" -doctype[2].fieldsets{[document]}.fields[1] "swag_field" -doctype[2].structtype[0].idx 10021 -doctype[2].structtype[0].name "campaign.header" -doctype[2].structtype[0].field[0].name "cool_field" -doctype[2].structtype[0].field[0].internalid 1588702436 +doctype[2].contentstruct 10018 +doctype[2].fieldsets{[document]}.fields[0] "name" +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "person.header" +doctype[2].structtype[0].field[0].name "name" +doctype[2].structtype[0].field[0].internalid 1160796772 doctype[2].structtype[0].field[0].type 10012 -doctype[2].structtype[0].field[1].name "swag_field" -doctype[2].structtype[0].field[1].internalid 1691224741 -doctype[2].structtype[0].field[1].type 10008 -doctype[2].structtype[0].internalid -2041471955 -doctype[3].name "person" -doctype[3].idx 10020 -doctype[3].internalid 443162583 +doctype[2].structtype[0].internalid 3129224 +doctype[3].name "ad" +doctype[3].idx 10019 +doctype[3].internalid 2987301 doctype[3].inherits[0].idx 10000 -doctype[3].contentstruct 10022 -doctype[3].fieldsets{[document]}.fields[0] "name" -doctype[3].structtype[0].idx 10022 -doctype[3].structtype[0].name "person.header" -doctype[3].structtype[0].field[0].name "name" -doctype[3].structtype[0].field[0].internalid 1160796772 -doctype[3].structtype[0].field[0].type 10012 -doctype[3].structtype[0].internalid 3129224 +doctype[3].contentstruct 10020 +doctype[3].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[3].fieldsets{[document]}.fields[1] "person_ref" +doctype[3].importedfield[0].name "my_cool_field" +doctype[3].importedfield[1].name "my_swag_field" +doctype[3].importedfield[2].name "my_name" +doctype[3].documentref[0].idx 10021 +doctype[3].documentref[0].targettype 10015 +doctype[3].documentref[0].internalid 595216861 +doctype[3].documentref[1].idx 10022 +doctype[3].documentref[1].targettype 10017 +doctype[3].documentref[1].internalid 542332920 +doctype[3].structtype[0].idx 10020 +doctype[3].structtype[0].name "ad.header" +doctype[3].structtype[0].field[0].name "campaign_ref" +doctype[3].structtype[0].field[0].internalid 23963250 +doctype[3].structtype[0].field[0].type 10021 +doctype[3].structtype[0].field[1].name "person_ref" +doctype[3].structtype[0].field[1].internalid 100779805 +doctype[3].structtype[0].field[1].type 10022 +doctype[3].structtype[0].internalid 959075962 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg index 1c5d4d41819..4222229cdd3 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg @@ -43,41 +43,41 @@ doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 doctype[0].structtype[1].internalid 1381038251 -doctype[1].name "ad" +doctype[1].name "campaign" doctype[1].idx 10015 -doctype[1].internalid 2987301 +doctype[1].internalid -1318255918 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" -doctype[1].fieldsets{[document]}.fields[1] "person_ref" -doctype[1].documentref[0].idx 10017 -doctype[1].documentref[0].targettype 10018 -doctype[1].documentref[0].internalid 595216861 -doctype[1].documentref[1].idx 10019 -doctype[1].documentref[1].targettype 10020 -doctype[1].documentref[1].internalid 542332920 doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "ad.header" -doctype[1].structtype[0].field[0].name "campaign_ref" -doctype[1].structtype[0].field[0].internalid 23963250 -doctype[1].structtype[0].field[0].type 10017 -doctype[1].structtype[0].field[1].name "person_ref" -doctype[1].structtype[0].field[1].internalid 100779805 -doctype[1].structtype[0].field[1].type 10019 -doctype[1].structtype[0].internalid 959075962 -doctype[2].name "campaign" -doctype[2].idx 10018 -doctype[2].internalid -1318255918 +doctype[1].structtype[0].name "campaign.header" +doctype[1].structtype[0].internalid -2041471955 +doctype[2].name "person" +doctype[2].idx 10017 +doctype[2].internalid 443162583 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10021 -doctype[2].structtype[0].idx 10021 -doctype[2].structtype[0].name "campaign.header" -doctype[2].structtype[0].internalid -2041471955 -doctype[3].name "person" -doctype[3].idx 10020 -doctype[3].internalid 443162583 +doctype[2].contentstruct 10018 +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "person.header" +doctype[2].structtype[0].internalid 3129224 +doctype[3].name "ad" +doctype[3].idx 10019 +doctype[3].internalid 2987301 doctype[3].inherits[0].idx 10000 -doctype[3].contentstruct 10022 -doctype[3].structtype[0].idx 10022 -doctype[3].structtype[0].name "person.header" -doctype[3].structtype[0].internalid 3129224 +doctype[3].contentstruct 10020 +doctype[3].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[3].fieldsets{[document]}.fields[1] "person_ref" +doctype[3].documentref[0].idx 10021 +doctype[3].documentref[0].targettype 10015 +doctype[3].documentref[0].internalid 595216861 +doctype[3].documentref[1].idx 10022 +doctype[3].documentref[1].targettype 10017 +doctype[3].documentref[1].internalid 542332920 +doctype[3].structtype[0].idx 10020 +doctype[3].structtype[0].name "ad.header" +doctype[3].structtype[0].field[0].name "campaign_ref" +doctype[3].structtype[0].field[0].internalid 23963250 +doctype[3].structtype[0].field[0].type 10021 +doctype[3].structtype[0].field[1].name "person_ref" +doctype[3].structtype[0].field[1].internalid 100779805 +doctype[3].structtype[0].field[1].type 10022 +doctype[3].structtype[0].internalid 959075962 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg index 2f178c55bfd..0d3802bcbe3 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg @@ -43,30 +43,30 @@ doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 doctype[0].structtype[1].internalid 1381038251 -doctype[1].name "ad" +doctype[1].name "campaign" doctype[1].idx 10015 -doctype[1].internalid 2987301 +doctype[1].internalid -1318255918 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" -doctype[1].fieldsets{[document]}.fields[1] "other_campaign_ref" -doctype[1].documentref[0].idx 10017 -doctype[1].documentref[0].targettype 10018 -doctype[1].documentref[0].internalid 595216861 doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "ad.header" -doctype[1].structtype[0].field[0].name "campaign_ref" -doctype[1].structtype[0].field[0].internalid 23963250 -doctype[1].structtype[0].field[0].type 10017 -doctype[1].structtype[0].field[1].name "other_campaign_ref" -doctype[1].structtype[0].field[1].internalid 874751172 -doctype[1].structtype[0].field[1].type 10017 -doctype[1].structtype[0].internalid 959075962 -doctype[2].name "campaign" -doctype[2].idx 10018 -doctype[2].internalid -1318255918 +doctype[1].structtype[0].name "campaign.header" +doctype[1].structtype[0].internalid -2041471955 +doctype[2].name "ad" +doctype[2].idx 10017 +doctype[2].internalid 2987301 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10019 -doctype[2].structtype[0].idx 10019 -doctype[2].structtype[0].name "campaign.header" -doctype[2].structtype[0].internalid -2041471955 +doctype[2].contentstruct 10018 +doctype[2].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[2].fieldsets{[document]}.fields[1] "other_campaign_ref" +doctype[2].documentref[0].idx 10019 +doctype[2].documentref[0].targettype 10015 +doctype[2].documentref[0].internalid 595216861 +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "ad.header" +doctype[2].structtype[0].field[0].name "campaign_ref" +doctype[2].structtype[0].field[0].internalid 23963250 +doctype[2].structtype[0].field[0].type 10019 +doctype[2].structtype[0].field[1].name "other_campaign_ref" +doctype[2].structtype[0].field[1].internalid 874751172 +doctype[2].structtype[0].field[1].type 10019 +doctype[2].structtype[0].internalid 959075962 diff --git a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg index 3f0994cf2bd..38298feaa0c 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg @@ -11,3 +11,12 @@ classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].id 659145226 +classes[].name "rename" +classes[].omitsummaryfeatures false +classes[].fields[].name "new_elem_array" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg index bb99d6ced39..9dd63a3f316 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg @@ -1,10 +1,13 @@ defaultoutputclass -1 -override[].field "elem_array" +override[].field "new_elem_array" override[].command "attributecombiner" -override[].arguments "" +override[].arguments "elem_array" override[].field "rankfeatures" override[].command "rankfeatures" override[].arguments "" override[].field "summaryfeatures" override[].command "summaryfeatures" override[].arguments "" +override[].field "elem_array" +override[].command "attributecombiner" +override[].arguments "" diff --git a/config-model/src/test/derived/array_of_struct_attribute/test.sd b/config-model/src/test/derived/array_of_struct_attribute/test.sd index 2f7c7755ca5..969dba96bf8 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/test.sd +++ b/config-model/src/test/derived/array_of_struct_attribute/test.sd @@ -16,4 +16,7 @@ schema test { } } } + document-summary rename { + summary new_elem_array type array<elem> { source: elem_array } + } } diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg index 7a67640adfe..b15734169a5 100644 --- a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg +++ b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg @@ -43,57 +43,57 @@ doctype[].structtype[].field[].name "y" doctype[].structtype[].field[].internalid 900009410 doctype[].structtype[].field[].type 10007 doctype[].structtype[].internalid 1381038251 -doctype[].name "child_a" +doctype[].name "parent" doctype[].idx 10015 -doctype[].internalid -94853056 +doctype[].internalid 1175161836 doctype[].inherits[].idx 10000 doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "int_field" +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "parent.header" +doctype[].structtype[].field[].name "int_field" +doctype[].structtype[].field[].internalid 2128822283 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].internalid 836075987 +doctype[].name "child_a" +doctype[].idx 10017 +doctype[].internalid -94853056 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10018 doctype[].fieldsets{[document]}.fields[] "ref_from_a" -doctype[].documentref[].idx 10017 -doctype[].documentref[].targettype 10018 +doctype[].documentref[].idx 10019 +doctype[].documentref[].targettype 10015 doctype[].documentref[].internalid 427398467 -doctype[].structtype[].idx 10016 +doctype[].structtype[].idx 10018 doctype[].structtype[].name "child_a.header" doctype[].structtype[].field[].name "ref_from_a" doctype[].structtype[].field[].internalid 300427062 -doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].type 10019 doctype[].structtype[].internalid 867409663 doctype[].name "child_b" -doctype[].idx 10019 +doctype[].idx 10020 doctype[].internalid -94852095 doctype[].inherits[].idx 10000 -doctype[].inherits[].idx 10015 -doctype[].contentstruct 10020 +doctype[].inherits[].idx 10017 +doctype[].contentstruct 10021 doctype[].fieldsets{[document]}.fields[] "ref_from_a" doctype[].fieldsets{[document]}.fields[] "ref_from_b" -doctype[].structtype[].idx 10020 +doctype[].structtype[].idx 10021 doctype[].structtype[].name "child_b.header" doctype[].structtype[].field[].name "ref_from_b" doctype[].structtype[].field[].internalid 185778735 -doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].type 10019 doctype[].structtype[].internalid 670896158 doctype[].name "child_c" -doctype[].idx 10021 +doctype[].idx 10022 doctype[].internalid -94851134 doctype[].inherits[].idx 10000 -doctype[].inherits[].idx 10019 -doctype[].contentstruct 10022 +doctype[].inherits[].idx 10020 +doctype[].contentstruct 10023 doctype[].fieldsets{[document]}.fields[] "ref_from_a" doctype[].fieldsets{[document]}.fields[] "ref_from_b" doctype[].importedfield[].name "from_a_int_field" doctype[].importedfield[].name "from_b_int_field" -doctype[].structtype[].idx 10022 +doctype[].structtype[].idx 10023 doctype[].structtype[].name "child_c.header" doctype[].structtype[].internalid 474382653 -doctype[].name "parent" -doctype[].idx 10018 -doctype[].internalid 1175161836 -doctype[].inherits[].idx 10000 -doctype[].contentstruct 10023 -doctype[].fieldsets{[document]}.fields[] "int_field" -doctype[].structtype[].idx 10023 -doctype[].structtype[].name "parent.header" -doctype[].structtype[].field[].name "int_field" -doctype[].structtype[].field[].internalid 2128822283 -doctype[].structtype[].field[].type 10007 -doctype[].structtype[].internalid 836075987 diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg index dadf7b5cfe7..997743389c6 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg @@ -1,6 +1,6 @@ -defaultsummaryid 1131098132 +defaultsummaryid 1048168773 usev8geopositions true -classes[].id 1131098132 +classes[].id 1048168773 classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "str_elem_map" @@ -11,5 +11,18 @@ classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].name "new_int_elem_map" +classes[].fields[].type "jsonstring" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].id 1424421039 +classes[].name "rename" +classes[].omitsummaryfeatures false +classes[].fields[].name "new_str_elem_map" +classes[].fields[].type "jsonstring" +classes[].fields[].name "new_int_elem_map" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg index 1540b821ae1..291dbb91542 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg @@ -1,10 +1,16 @@ defaultoutputclass -1 -override[].field "str_elem_map" +override[].field "new_str_elem_map" override[].command "attributecombiner" -override[].arguments "" +override[].arguments "str_elem_map" +override[].field "new_int_elem_map" +override[].command "copy" +override[].arguments "int_elem_map" override[].field "rankfeatures" override[].command "rankfeatures" override[].arguments "" override[].field "summaryfeatures" override[].command "summaryfeatures" override[].arguments "" +override[].field "str_elem_map" +override[].command "attributecombiner" +override[].arguments "" diff --git a/config-model/src/test/derived/map_of_struct_attribute/test.sd b/config-model/src/test/derived/map_of_struct_attribute/test.sd index 7806d49392e..2e71678148b 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/test.sd +++ b/config-model/src/test/derived/map_of_struct_attribute/test.sd @@ -29,4 +29,8 @@ schema test { } } } + document-summary rename { + summary new_str_elem_map type map<string,elem> { source: str_elem_map } + summary new_int_elem_map type map<int,elem> { source: int_elem_map } + } } diff --git a/config-model/src/test/derived/multiplesummaries/juniperrc.cfg b/config-model/src/test/derived/multiplesummaries/juniperrc.cfg index 9b5c6a5a7ba..13d68f62e8b 100644 --- a/config-model/src/test/derived/multiplesummaries/juniperrc.cfg +++ b/config-model/src/test/derived/multiplesummaries/juniperrc.cfg @@ -9,33 +9,33 @@ max_match_candidates 1000 stem_min_length 5 stem_max_extend 3 override[].fieldname "d" -override[].length 65536 +override[].length 67108864 override[].max_matches 1 override[].min_length 8192 override[].prefix true -override[].surround_max 65536 +override[].surround_max 67108864 override[].winsize 200 override[].winsize_fallback_multiplier 10.0 override[].max_match_candidates 1000 override[].stem_min_length 5 override[].stem_max_extend 3 override[].fieldname "abolded2" -override[].length 65536 +override[].length 67108864 override[].max_matches 1 override[].min_length 8192 override[].prefix true -override[].surround_max 65536 +override[].surround_max 67108864 override[].winsize 200 override[].winsize_fallback_multiplier 10.0 override[].max_match_candidates 1000 override[].stem_min_length 5 override[].stem_max_extend 3 override[].fieldname "abolded" -override[].length 65536 +override[].length 67108864 override[].max_matches 1 override[].min_length 8192 override[].prefix true -override[].surround_max 65536 +override[].surround_max 67108864 override[].winsize 200 override[].winsize_fallback_multiplier 10.0 override[].max_match_candidates 1000 diff --git a/config-model/src/test/derived/music/juniperrc.cfg b/config-model/src/test/derived/music/juniperrc.cfg index f5b2c166687..ec5cb1b6a0d 100644 --- a/config-model/src/test/derived/music/juniperrc.cfg +++ b/config-model/src/test/derived/music/juniperrc.cfg @@ -9,11 +9,11 @@ max_match_candidates 1000 stem_min_length 5 stem_max_extend 3 override[].fieldname "ew" -override[].length 65536 +override[].length 67108864 override[].max_matches 1 override[].min_length 8192 override[].prefix true -override[].surround_max 65536 +override[].surround_max 67108864 override[].winsize 200 override[].winsize_fallback_multiplier 10.0 override[].max_match_candidates 1000 diff --git a/config-model/src/test/derived/newrank/juniperrc.cfg b/config-model/src/test/derived/newrank/juniperrc.cfg index f5b2c166687..ec5cb1b6a0d 100644 --- a/config-model/src/test/derived/newrank/juniperrc.cfg +++ b/config-model/src/test/derived/newrank/juniperrc.cfg @@ -9,11 +9,11 @@ max_match_candidates 1000 stem_min_length 5 stem_max_extend 3 override[].fieldname "ew" -override[].length 65536 +override[].length 67108864 override[].max_matches 1 override[].min_length 8192 override[].prefix true -override[].surround_max 65536 +override[].surround_max 67108864 override[].winsize 200 override[].winsize_fallback_multiplier 10.0 override[].max_match_candidates 1000 diff --git a/config-model/src/test/derived/reference_from_several/documentmanager.cfg b/config-model/src/test/derived/reference_from_several/documentmanager.cfg index 0418b05d9d6..6ac1aab0baf 100644 --- a/config-model/src/test/derived/reference_from_several/documentmanager.cfg +++ b/config-model/src/test/derived/reference_from_several/documentmanager.cfg @@ -39,45 +39,45 @@ doctype[].structtype[].field[].type 10007 doctype[].structtype[].field[].name "y" doctype[].structtype[].field[].internalid 900009410 doctype[].structtype[].field[].type 10007 -doctype[].name "bar" +doctype[].name "parent" doctype[].idx 10015 doctype[].inherits[].idx 10000 doctype[].contentstruct 10016 -doctype[].fieldsets{[document]}.fields[] "bartitle" -doctype[].fieldsets{[document]}.fields[] "bpref" -doctype[].importedfield[].name "barsximp" -doctype[].documentref[].idx 10017 -doctype[].documentref[].targettype 10018 +doctype[].fieldsets{[document]}.fields[] "x" doctype[].structtype[].idx 10016 -doctype[].structtype[].name "bar.header" -doctype[].structtype[].field[].name "bpref" -doctype[].structtype[].field[].internalid 1709838545 -doctype[].structtype[].field[].type 10017 -doctype[].structtype[].field[].name "bartitle" -doctype[].structtype[].field[].internalid 1554393914 -doctype[].structtype[].field[].type 10012 +doctype[].structtype[].name "parent.header" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 doctype[].name "foo" -doctype[].idx 10019 +doctype[].idx 10017 doctype[].inherits[].idx 10000 -doctype[].contentstruct 10020 +doctype[].contentstruct 10018 doctype[].fieldsets{[document]}.fields[] "foo" doctype[].fieldsets{[document]}.fields[] "myref" doctype[].importedfield[].name "myx" -doctype[].structtype[].idx 10020 +doctype[].documentref[].idx 10019 +doctype[].documentref[].targettype 10015 +doctype[].structtype[].idx 10018 doctype[].structtype[].name "foo.header" doctype[].structtype[].field[].name "myref" doctype[].structtype[].field[].internalid 598565475 -doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].type 10019 doctype[].structtype[].field[].name "foo" doctype[].structtype[].field[].internalid 846279091 doctype[].structtype[].field[].type 10012 -doctype[].name "parent" -doctype[].idx 10018 +doctype[].name "bar" +doctype[].idx 10020 doctype[].inherits[].idx 10000 doctype[].contentstruct 10021 -doctype[].fieldsets{[document]}.fields[] "x" +doctype[].fieldsets{[document]}.fields[] "bartitle" +doctype[].fieldsets{[document]}.fields[] "bpref" +doctype[].importedfield[].name "barsximp" doctype[].structtype[].idx 10021 -doctype[].structtype[].name "parent.header" -doctype[].structtype[].field[].name "x" -doctype[].structtype[].field[].internalid 914677694 -doctype[].structtype[].field[].type 10007 +doctype[].structtype[].name "bar.header" +doctype[].structtype[].field[].name "bpref" +doctype[].structtype[].field[].internalid 1709838545 +doctype[].structtype[].field[].type 10019 +doctype[].structtype[].field[].name "bartitle" +doctype[].structtype[].field[].internalid 1554393914 +doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/examples/fieldoftypedocument.cfg b/config-model/src/test/examples/fieldoftypedocument.cfg index ea56800ea1c..b59b3206a2d 100644 --- a/config-model/src/test/examples/fieldoftypedocument.cfg +++ b/config-model/src/test/examples/fieldoftypedocument.cfg @@ -39,31 +39,31 @@ doctype[0].structtype[1].field[0].type 10007 doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 -doctype[1].name "book" +doctype[1].name "music" doctype[1].idx 10015 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "soundtrack" +doctype[1].fieldsets{[document]}.fields[0] "intfield" +doctype[1].fieldsets{[document]}.fields[1] "longfield" +doctype[1].fieldsets{[document]}.fields[2] "stringfield" doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "book.header" -doctype[1].structtype[0].field[0].name "soundtrack" -doctype[1].structtype[0].field[0].internalid 1258961213 -doctype[1].structtype[0].field[0].type 10017 -doctype[2].name "music" +doctype[1].structtype[0].name "music.header" +doctype[1].structtype[0].field[0].name "intfield" +doctype[1].structtype[0].field[0].internalid 435380425 +doctype[1].structtype[0].field[0].type 10007 +doctype[1].structtype[0].field[1].name "stringfield" +doctype[1].structtype[0].field[1].internalid 1182460484 +doctype[1].structtype[0].field[1].type 10012 +doctype[1].structtype[0].field[2].name "longfield" +doctype[1].structtype[0].field[2].internalid 1589309697 +doctype[1].structtype[0].field[2].type 10008 +doctype[2].name "book" doctype[2].idx 10017 doctype[2].inherits[0].idx 10000 doctype[2].contentstruct 10018 -doctype[2].fieldsets{[document]}.fields[0] "intfield" -doctype[2].fieldsets{[document]}.fields[1] "longfield" -doctype[2].fieldsets{[document]}.fields[2] "stringfield" +doctype[2].fieldsets{[document]}.fields[0] "soundtrack" doctype[2].structtype[0].idx 10018 -doctype[2].structtype[0].name "music.header" -doctype[2].structtype[0].field[0].name "intfield" -doctype[2].structtype[0].field[0].internalid 435380425 -doctype[2].structtype[0].field[0].type 10007 -doctype[2].structtype[0].field[1].name "stringfield" -doctype[2].structtype[0].field[1].internalid 1182460484 -doctype[2].structtype[0].field[1].type 10012 -doctype[2].structtype[0].field[2].name "longfield" -doctype[2].structtype[0].field[2].internalid 1589309697 -doctype[2].structtype[0].field[2].type 10008 +doctype[2].structtype[0].name "book.header" +doctype[2].structtype[0].field[0].name "soundtrack" +doctype[2].structtype[0].field[0].internalid 1258961213 +doctype[2].structtype[0].field[0].type 10015 diff --git a/config-model/src/test/java/com/yahoo/schema/SDDocumentTypeOrdererTestCase.java b/config-model/src/test/java/com/yahoo/schema/SDDocumentTypeOrdererTestCase.java deleted file mode 100755 index 45780d39021..00000000000 --- a/config-model/src/test/java/com/yahoo/schema/SDDocumentTypeOrdererTestCase.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.document.DataType; -import com.yahoo.document.DataTypeName; -import com.yahoo.schema.document.SDDocumentType; -import com.yahoo.schema.document.SDField; -import com.yahoo.schema.document.TemporarySDDocumentType; -import com.yahoo.schema.document.TemporarySDField; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author Einar M R Rosenvinge - */ -public class SDDocumentTypeOrdererTestCase { - - @Test - public void testOrder() { - List<SDDocumentType> types = new ArrayList<>(); - - SDDocumentType a = new SDDocumentType("a"); - SDDocumentType b = new SDDocumentType("b"); - SDDocumentType c = new SDDocumentType("c"); - SDDocumentType d = new SDDocumentType("d"); - SDDocumentType e = new SDDocumentType("e"); - SDDocumentType f = new SDDocumentType("f"); - SDDocumentType g = new SDDocumentType("g"); - b.inherit(new TemporarySDDocumentType(new DataTypeName("a"))); - c.inherit(new TemporarySDDocumentType(new DataTypeName("b"))); - d.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); - g.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); - g.inherit(new TemporarySDDocumentType(new DataTypeName("c"))); - - SDField aFieldTypeB = new TemporarySDField(a, "atypeb", DataType.STRING); - a.addField(aFieldTypeB); - - SDField bFieldTypeC = new TemporarySDField(b, "btypec", DataType.STRING); - b.addField(bFieldTypeC); - - SDField cFieldTypeG = new TemporarySDField(c, "ctypeg", DataType.STRING); - c.addField(cFieldTypeG); - - SDField gFieldTypeF = new TemporarySDField(g, "gtypef", DataType.STRING); - g.addField(gFieldTypeF); - - SDField fFieldTypeC = new TemporarySDField(f, "ftypec", DataType.STRING); - f.addField(fFieldTypeC); - - SDField dFieldTypeE = new TemporarySDField(d, "dtypee", DataType.STRING); - d.addField(dFieldTypeE); - - types.add(a); - types.add(b); - types.add(c); - types.add(d); - types.add(e); - types.add(f); - types.add(g); - - SDDocumentTypeOrderer app = new SDDocumentTypeOrderer(types, new BaseDeployLogger()); - app.process(); - assertEquals(7, app.processingOrder.size()); - assertEquals(a, app.processingOrder.get(0)); - assertEquals(b, app.processingOrder.get(1)); - assertEquals(c, app.processingOrder.get(2)); - assertEquals(e, app.processingOrder.get(3)); - assertEquals(d, app.processingOrder.get(4)); - assertEquals(f, app.processingOrder.get(5)); - assertEquals(g, app.processingOrder.get(6)); - } - -} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java index 16a9a459dcb..dc10004a631 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java @@ -115,7 +115,7 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testAvoidRenamingRankingExpression() throws IOException, ParseException { assertCorrectDeriving("renamedfeatures", "foo", - new TestProperties().setAvoidRenamingSummaryFeatures(true), + new TestProperties(), new TestableDeployLogger()); } diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java index e672763f13c..4683bccb0ad 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java @@ -75,6 +75,7 @@ public class SchemaOrdererTestCase extends AbstractSchemaTestCase { } private static void assertOrder(List<String> expectedSearchOrder, List<String> inputNames) { + inputNames.sort((a, b) -> a.compareTo(b)); Map<String, Schema> schemas = createSchemas(); List<Schema> inputSchemas = inputNames.stream() .map(schemas::get) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 16ab941d3be..8750d3caa47 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -21,7 +21,7 @@ public class QuotaValidatorTest { private final Zone publicZone = new Zone(SystemName.Public, Environment.prod, RegionName.from("foo")); private final Zone publicCdZone = new Zone(SystemName.PublicCd, Environment.prod, RegionName.from("foo")); - private final Quota quota = Quota.unlimited().withClusterSize(10).withBudget(BigDecimal.valueOf(1)); + private final Quota quota = Quota.unlimited().withClusterSize(10).withBudget(BigDecimal.valueOf(1.25)); @Test public void test_deploy_under_quota() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index d23dca65dd1..635f799411b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -291,7 +291,7 @@ public class DistributorTest { cluster.getChildren().get("0").getConfig(builder); StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); assertTrue(config.mbus().dispatch_on_encode()); - assertFalse(config.mbus().dispatch_on_decode()); + assertTrue(config.mbus().dispatch_on_decode()); assertEquals(4, config.mbus().num_threads()); assertEquals(StorCommunicationmanagerConfig.Mbus.Optimize_for.LATENCY, config.mbus().optimize_for()); assertFalse(config.skip_thread()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 8908ab9f5b9..c03ffba1c34 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -101,8 +101,8 @@ public class StorageClusterTest { StorCommunicationmanagerConfig.Builder builder = new StorCommunicationmanagerConfig.Builder(); storage.getChildren().get("0").getConfig(builder); StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); - assertFalse(config.mbus().dispatch_on_encode()); - assertFalse(config.mbus().dispatch_on_decode()); + assertTrue(config.mbus().dispatch_on_encode()); + assertTrue(config.mbus().dispatch_on_decode()); assertEquals(4, config.mbus().num_threads()); assertEquals(StorCommunicationmanagerConfig.Mbus.Optimize_for.LATENCY, config.mbus().optimize_for()); assertFalse(config.skip_thread()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index d541a6422e7..a3c57dae2de 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search.test; import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.path.Path; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; @@ -101,6 +100,24 @@ public class DocumentDatabaseTestCase { assertEquals(expectedConcurrency, proton.feeding().concurrency(), SMALL); } + private void verifyFeedNiceness(List<DocType> nameAndModes, Double expectedNiceness, Double featureFlagNiceness) { + TestProperties properties = new TestProperties(); + if (featureFlagNiceness != null) { + properties.setFeedNiceness(featureFlagNiceness); + } + var tester = new SchemaTester(); + VespaModel model = tester.createModel(nameAndModes, "", new DeployState.Builder().properties(properties)); + ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); + ProtonConfig proton = tester.getProtonConfig(contentSearchCluster); + assertEquals(expectedNiceness, proton.feeding().niceness(), SMALL); + } + + @Test + public void requireFeedNicenessIsReflected() { + verifyFeedNiceness(Arrays.asList(DocType.create("a", "index")), 0.0, null); + verifyFeedNiceness(Arrays.asList(DocType.create("a", "index")), 0.32, 0.32); + } + @Test public void requireThatModeIsSet() { var tester = new SchemaTester(); @@ -262,36 +279,8 @@ public class DocumentDatabaseTestCase { { // documentdb-info config DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, searcherId); assertEquals(2, dcfg.documentdb().size()); - - { // type1 - DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(0); - assertEquals("type1", db.name()); - - assertEquals(7, db.rankprofile().size()); - tester.assertRankProfile(db, 0, "default", false, false); - tester.assertRankProfile(db, 1, "unranked", false, false); - tester.assertRankProfile(db, 2, "staticrank", false, false); - tester.assertRankProfile(db, 3, "summaryfeatures", true, false); - tester.assertRankProfile(db, 4, "inheritedsummaryfeatures", true, false); - tester.assertRankProfile(db, 5, "rankfeatures", false, true); - var inputs = tester.assertRankProfile(db, 6, "inputs", false, false); - - assertEquals(2, inputs.input().size()); - assertEquals("query(foo)", inputs.input(0).name()); - assertEquals("tensor<float>(x[10])", inputs.input(0).type()); - assertEquals("query(bar)", inputs.input(1).name()); - assertEquals("tensor(key{},x[1000])", inputs.input(1).type()); - - assertEquals(2, db.summaryclass().size()); - assertEquals("default", db.summaryclass(0).name()); - assertEquals("attributeprefetch", db.summaryclass(1).name()); - tester.assertSummaryField(db, 0, 0, "f1", "longstring", true); - tester.assertSummaryField(db, 0, 1, "f2", "integer", false); - } - { // type2 - DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(1); - assertEquals("type2", db.name()); - } + assertEquals("type1", dcfg.documentdb(0).name()); + assertEquals("type2", dcfg.documentdb(1).name()); } { // attributes config AttributesConfig acfg = model.getConfig(AttributesConfig.class, searcherId); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java index 6b740816775..9d1aece61d4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java @@ -2,9 +2,7 @@ package com.yahoo.vespa.model.search.test; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.path.Path; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.VespaModel; @@ -200,18 +198,6 @@ public class SchemaTester { return schemas; } - DocumentdbInfoConfig.Documentdb.Rankprofile assertRankProfile(DocumentdbInfoConfig.Documentdb db, - int index, - String name, - boolean hasSummaryFeatures, - boolean hasRankFeatures) { - DocumentdbInfoConfig.Documentdb.Rankprofile rankProfile = db.rankprofile(index); - assertEquals(name, rankProfile.name()); - assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); - assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); - return rankProfile; - } - SchemaInfoConfig.Schema.Rankprofile assertRankProfile(SchemaInfoConfig.Schema schema, int index, String name, @@ -224,14 +210,6 @@ public class SchemaTester { return rankProfile; } - void assertSummaryField(DocumentdbInfoConfig.Documentdb db, int summaryClassIndex, int fieldIndex, - String name, String type, boolean dynamic) { - DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field = db.summaryclass(summaryClassIndex).fields(fieldIndex); - assertEquals(name, field.name()); - assertEquals(type, field.type()); - assertEquals(dynamic, field.dynamic()); - } - void assertSummaryField(SchemaInfoConfig.Schema schema, int summaryClassIndex, int fieldIndex, String name, String type, boolean dynamic) { SchemaInfoConfig.Schema.Summaryclass.Fields field = schema.summaryclass(summaryClassIndex).fields(fieldIndex); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Deployer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Deployer.java index 209f339f51f..1498b9871d3 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Deployer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Deployer.java @@ -54,7 +54,7 @@ public interface Deployer { */ Optional<Deployment> deployFromLocalActive(ApplicationId application, Duration timeout, boolean bootstrap); - /** Returns the time the current local active session was created, or empty if there is no local active session */ + /** Returns the time the current local active session was activated, or empty if there is no local active session */ Optional<Instant> lastDeployTime(ApplicationId application); /** Whether the deployer is bootstrapping, some users of the deployer will want to hold off with deployments in that case. */ diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 21349910ca1..25771f1906b 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -12,9 +12,9 @@ import java.util.Optional; public class NodeResources { // Standard unit cost in dollars per hour - private static final double cpuUnitCost = 0.09; - private static final double memoryUnitCost = 0.009; - private static final double diskUnitCost = 0.0003; + private static final double cpuUnitCost = 0.11; + private static final double memoryUnitCost = 0.011; + private static final double diskUnitCost = 0.0004; private static final NodeResources zero = new NodeResources(0, 0, 0, 0); private static final NodeResources unspecified = new NodeResources(0, 0, 0, 0); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java index 058be998478..e65340aa59b 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java @@ -1,13 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision.zone; -import com.yahoo.config.provision.CloudName; - /** * A ZoneId list which can be filtered in various ways; elements can be accessed after at least one filter. * * The methods here return instances of {@link ZoneList}, which extends ZoneFilter, but with accessors and additional filters. * This forces the developer to consider which of the filters in this class to apply, prior to accessing any zones. + * Note: Do not add further filters, as this is only meant for the levels of configuration of the zone, not other properties. * * @author jonmv */ @@ -16,24 +15,13 @@ public interface ZoneFilter { /** Negates the next filter. */ ZoneFilter not(); - /** Zones which are upgraded by the controller. */ - ZoneList controllerUpgraded(); - - /** Zones where traffic is routed using given method */ - ZoneList routingMethod(RoutingMethod method); - /** Zones where config servers are up and running. */ ZoneList reachable(); - /** Zones where hosts must be reprovisioned to upgrade their OS */ - ZoneList reprovisionToUpgradeOs(); + /** Zones which are upgraded by the controller. */ + ZoneList controllerUpgraded(); /** All zones from the initial pool. */ ZoneList all(); - /** Zones in the specified cloud */ - default ZoneList ofCloud(CloudName cloud) { - return all(); // Not implemented in this repo. - } - } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java index c6ace00b90c..0a6bdd3b6b8 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java @@ -1,10 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision.zone; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; /** @@ -27,9 +29,23 @@ public interface ZoneList extends ZoneFilter { /** Zones in one of the given regions. */ ZoneList in(RegionName... regions); + /** Zones in one of the given clouds. */ + ZoneList in(CloudName... clouds); + /** Only the given zones — combine with not() for best effect! */ ZoneList among(ZoneId... zones); + /** Zones where hosts must be reprovisioned to upgrade their OS */ + ZoneList reprovisionToUpgradeOs(); + + /** Zones where traffic is routed using given method */ + ZoneList routingMethod(RoutingMethod method); + + /** Returns the zone with the given id, if this exists. */ + default Optional<? extends ZoneApi> get(ZoneId id) { + return among(id).zones().stream().findFirst(); + } + /** Returns the ZoneApi of all zones in this list. */ List<? extends ZoneApi> zones(); diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterResourcesTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterResourcesTest.java index e342a90c957..cbca931e4d0 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterResourcesTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterResourcesTest.java @@ -14,7 +14,7 @@ public class ClusterResourcesTest { public void testCost() { ClusterResources r1 = new ClusterResources(3, 1, new NodeResources(2, 8, 50, 1)); ClusterResources r2 = new ClusterResources(3, 1, new NodeResources(2, 16, 50, 1)); - assertEquals(1.818, r1.cost() + r2.cost(), 0.01); + assertEquals(2.232, r1.cost() + r2.cost(), 0.01); } } diff --git a/config/src/tests/misc/configsystem.cpp b/config/src/tests/misc/configsystem.cpp index db3d0a80706..db6764e6b17 100644 --- a/config/src/tests/misc/configsystem.cpp +++ b/config/src/tests/misc/configsystem.cpp @@ -5,6 +5,7 @@ #include <vespa/defaults.h> #include <vespa/fastos/file.h> #include <unistd.h> +#include <filesystem> using namespace config; @@ -25,23 +26,20 @@ TEST("require that bad home directory fails") { TEST("require that incorrect pid file type fails") { ASSERT_TRUE(nullptr != getcwd(cwd, sizeof(cwd))); - FastOS_File::EmptyAndRemoveDirectory("var"); - FastOS_File::MakeDirIfNotPresentOrExit("var"); - FastOS_File::MakeDirIfNotPresentOrExit("var/run"); - FastOS_File::MakeDirIfNotPresentOrExit("var/run/configproxy.pid"); + std::filesystem::remove_all(std::filesystem::path("var")); + std::filesystem::create_directories(std::filesystem::path("var/run/configproxy.pid")); ASSERT_EQUAL(0, setenv(VESPA_HOME, cwd, 1)); vespa::Defaults::bootstrap(cwd); ConfigSystem configSystem; ASSERT_FALSE(configSystem.isUp()); - FastOS_File::EmptyAndRemoveDirectory("var"); + std::filesystem::remove_all(std::filesystem::path("var")); } TEST("require that correct pid file succeeds") { ASSERT_TRUE(nullptr != getcwd(cwd, sizeof(cwd))); - FastOS_File::EmptyAndRemoveDirectory("var"); - FastOS_File::MakeDirIfNotPresentOrExit("var"); - FastOS_File::MakeDirIfNotPresentOrExit("var/run"); + std::filesystem::remove_all(std::filesystem::path("var")); + std::filesystem::create_directories(std::filesystem::path("var/run")); FastOS_File pid_file("var/run/configproxy.pid"); pid_file.OpenWriteOnlyTruncate(); ASSERT_TRUE(pid_file.Close()); @@ -50,7 +48,7 @@ TEST("require that correct pid file succeeds") { vespa::Defaults::bootstrap(cwd); ConfigSystem configSystem; ASSERT_TRUE(configSystem.isUp()); - FastOS_File::EmptyAndRemoveDirectory("var"); + std::filesystem::remove_all(std::filesystem::path("var")); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java index 2560b218e54..fe1566aa133 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java @@ -47,7 +47,6 @@ public class JavaClassBuilder implements ClassBuilder { try (PrintStream out = new PrintStream(new FileOutputStream(outFile))) { out.print(getConfigClass(className)); } - System.err.println(outFile.getPath() + " successfully written."); } catch (FileNotFoundException e) { throw new CodegenRuntimeException(e); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index fd4f1824885..ca56a200c2c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -437,7 +437,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public Optional<Instant> lastDeployTime(ApplicationId application) { Tenant tenant = tenantRepository.getTenant(application.tenant()); if (tenant == null) return Optional.empty(); - return getActiveSession(tenant, application).map(Session::getCreateTime); + Optional<Instant> activatedTime = getActiveSession(tenant, application).map(Session::getActivatedTime); + log.log(Level.FINE, application + " last activated " + activatedTime.orElse(Instant.EPOCH)); + return activatedTime; } public ApplicationId activate(Tenant tenant, @@ -750,7 +752,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public HttpResponse getLogs(ApplicationId applicationId, Optional<DomainName> hostname, String apiParams) { String logServerURI = getLogServerURI(applicationId, hostname) + apiParams; - return logRetriever.getLogs(logServerURI); + return logRetriever.getLogs(logServerURI, lastDeployTime(applicationId)); } // ---------------- Methods to do call against tester containers in hosted ------------------------------ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java index 6d6b93dad46..0ec1382b496 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java @@ -24,7 +24,7 @@ public class ServerCache { private final ConfigDefinitionRepo builtinConfigDefinitions; private final ConfigDefinitionRepo userConfigDefinitions; - // NOTE: The reason we do a double mapping here is to de-dupe configs that have the same md5. + // NOTE: The reason we do a double mapping here is to de-dupe configs that have the same checksum. private final Map<ConfigCacheKey, PayloadChecksum> checksums = new ConcurrentHashMap<>(); private final Map<PayloadChecksum, ConfigResponse> checksumToConfig = new ConcurrentHashMap<>(); private final Object [] stripedLocks = new Object[113]; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 3cc3e749904..1aa70ff4b5b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -206,7 +206,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { } else { deployLogger.log(Level.INFO, "Services did not converge on new config generation " + response.wantedGeneration + ", current generation: " + response.currentGeneration + ", will retry"); - try { Thread.sleep(10_000); } catch (InterruptedException e) { /* ignore */ } + try { Thread.sleep(5_000); } catch (InterruptedException e) { /* ignore */ } } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 30116b7ee69..c31d4603353 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -176,6 +176,7 @@ public class ModelContextImpl implements ModelContext { private final boolean skipMbusReplyThread; private final boolean useAsyncMessageHandlingOnSchedule; private final double feedConcurrency; + private final double feedNiceness; private final List<String> allowedAthenzProxyIdentities; private final int maxActivationInhibitedOutOfSyncGroups; private final ToIntFunction<ClusterSpec.Type> jvmOmitStackTraceInFastThrow; @@ -222,6 +223,7 @@ public class ModelContextImpl implements ModelContext { this.skipMbusReplyThread = flagValue(source, appId, version, Flags.SKIP_MBUS_REPLY_THREAD); this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, version, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); this.feedConcurrency = flagValue(source, appId, version, Flags.FEED_CONCURRENCY); + this.feedNiceness = flagValue(source, appId, version, Flags.FEED_NICENESS); this.allowedAthenzProxyIdentities = flagValue(source, appId, version, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, version, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS); this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, version, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); @@ -268,6 +270,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } + @Override public double feedNiceness() { return feedNiceness; } @Override public List<String> allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { @@ -380,7 +383,7 @@ public class ModelContextImpl implements ModelContext { private final Optional<CloudAccount> cloudAccount; public Properties(ApplicationId applicationId, - Version nodeVespaVersion, + Version modelVersion, ConfigserverConfig configserverConfig, Zone zone, Set<ContainerEndpoint> endpoints, @@ -394,7 +397,7 @@ public class ModelContextImpl implements ModelContext { SecretStore secretStore, List<X509Certificate> operatorCertificates, Optional<CloudAccount> cloudAccount) { - this.featureFlags = new FeatureFlags(flagSource, applicationId, nodeVespaVersion); + this.featureFlags = new FeatureFlags(flagSource, applicationId, modelVersion); this.applicationId = applicationId; this.multitenant = configserverConfig.multitenant() || configserverConfig.hostedVespa() || Boolean.getBoolean("multitenant"); this.configServerSpecs = fromConfig(configserverConfig); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java index 14586e2289e..3be99ab8393 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java @@ -6,8 +6,12 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.yolean.Exceptions; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; - import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; /** * @author olaaun @@ -16,11 +20,20 @@ public class LogRetriever { private final CloseableHttpClient httpClient = VespaHttpClientBuilder.create().build(); - public HttpResponse getLogs(String logServerUri) { + public HttpResponse getLogs(String logServerUri, Optional<Instant> deployTime) { HttpGet get = new HttpGet(logServerUri); try { return new ProxyResponse(httpClient.execute(get)); } catch (IOException e) { + // It takes some time before nodes are up after first-time deployment, return empty log for up to 2 minutes + // if getting logs fail + if (deployTime.isPresent() && Instant.now().isBefore(deployTime.get().plus(Duration.ofMinutes(2)))) + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("".getBytes(StandardCharsets.UTF_8)); + } + }; return HttpErrorResponse.internalServerError("Failed to get logs: " + Exceptions.toMessageString(e)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java index 3f2cd1e41af..d0c4f43359a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java @@ -26,6 +26,15 @@ public class SessionHandler extends HttpHandler { } /** + * Set to make sure that timeout for the handler is higher than any timeouts used inside the handler (e.g. zookeeper barrier timeout) + * Setting this too low will lead to a response with status code 504 and empty response body. + */ + @Override + public Duration getTimeout() { + return Duration.ofSeconds(applicationRepository.configserverConfig().zookeeper().barrierTimeout()).plus(Duration.ofSeconds(30)); + } + + /** * Gets the raw session id from request (v2). Input request must have a valid path. * * @param request a request diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index b489eb70927..ad68073053d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -100,7 +100,7 @@ public class ApplicationApiHandler extends SessionHandler { @Override public Duration getTimeout() { - return zookeeperBarrierTimeout.plus(Duration.ofSeconds(10)); + return zookeeperBarrierTimeout.plus(Duration.ofSeconds(30)); } private TenantName validateTenant(HttpRequest request) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java index 8f585dccf52..6244f806f47 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import java.time.Duration; - import com.yahoo.component.annotation.Inject; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.provision.ApplicationId; @@ -12,14 +10,15 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.ResponseHandler; -import java.util.logging.Level; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.http.v2.response.SessionActiveResponse; -import com.yahoo.vespa.config.server.tenant.Tenant; -import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.http.v2.response.SessionActiveResponse; +import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import java.time.Duration; +import java.util.logging.Level; /** * Handler that activates a session given by tenant and id (PUT). diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java index 620f76961d4..7fd94667159 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java @@ -53,11 +53,6 @@ public class SessionPrepareHandler extends SessionHandler { return new SessionPrepareResponse(tenant.getName(), request, sessionId); } - @Override - public Duration getTimeout() { - return zookeeperBarrierTimeout.plus(Duration.ofSeconds(10)); - } - private Tenant getExistingTenant(HttpRequest request) { TenantName tenantName = Utils.getTenantNameFromSessionRequest(request); Utils.checkThatTenantExists(tenantRepository, tenantName); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index b2915bc74fb..ae4b205c06e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -80,9 +80,9 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { false); if (fileDownloader.getFile(download).isEmpty()) { failures++; - log.info("Failed downloading application package (" + appFileReference + ")" + - " for " + applicationId + " (session " + - applicationRepository.getActiveSession(applicationId) + ")"); + log.info("Downloading application package (" + appFileReference + ")" + + " for " + applicationId + " (session " + sessionId + ") unsuccessful. " + + "Can be ignored unless it happens many times over a long period of time, retries is expected"); continue; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 468dda605c6..019b0386695 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -99,7 +99,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { Version wantedNodeVespaVersion) { log.log(Level.FINE, () -> String.format("Loading model version %s for session %s application %s", modelFactory.version(), applicationGeneration, applicationId)); - ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId, wantedNodeVespaVersion, applicationPackage); + ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId, modelFactory.version(), applicationPackage); Provisioned provisioned = new Provisioned(); ModelContext modelContext = new ModelContextImpl( applicationPackage, @@ -143,9 +143,11 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { return Optional.of(value); } - private ModelContext.Properties createModelContextProperties(ApplicationId applicationId, Version wantedNodeVespaVersion, ApplicationPackage applicationPackage) { + private ModelContext.Properties createModelContextProperties(ApplicationId applicationId, + Version modelVersion, + ApplicationPackage applicationPackage) { return new ModelContextImpl.Properties(applicationId, - wantedNodeVespaVersion, + modelVersion, configserverConfig, zone(), ImmutableSet.copyOf(new ContainerEndpointsCache(TenantRepository.getTenantPath(tenant), curator).read(applicationId)), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 361d966ebc5..cce6429f84a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -92,9 +92,6 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { Instant start = Instant.now(); log.log(Level.FINE, () -> "Will build models for " + applicationId); Set<Version> versions = modelFactoryRegistry.allVersions(); - if (applicationPackage.getMajorVersion().isPresent() && applicationPackage.getMajorVersion().get() != wantedNodeVespaVersion.getMajor()) - throw new IllegalArgumentException("requested node version (" + wantedNodeVespaVersion + ") has a different major version " + - "than specified in deployment.xml (" + applicationPackage.getMajorVersion().get() + ")"); // If the application specifies a major, skip models on a newer major Optional<Integer> requestedMajorVersion = applicationPackage.getMajorVersion(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index 51e61edfff3..e6ae5c491c0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -8,6 +8,8 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; @@ -22,6 +24,8 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase; import com.yahoo.vespa.config.server.application.ApplicationSet; @@ -31,6 +35,7 @@ import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.flags.FlagSource; import java.io.File; import java.io.IOException; @@ -38,6 +43,7 @@ import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.logging.Level; import java.util.logging.Logger; @@ -51,17 +57,24 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P private static final Logger log = Logger.getLogger(PreparedModelsBuilder.class.getName()); private final PermanentApplicationPackage permanentApplicationPackage; + private final FlagSource flagSource; + private final SecretStore secretStore; + private final List<ContainerEndpoint> containerEndpoints; + private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final ConfigDefinitionRepo configDefinitionRepo; private final HostValidator<ApplicationId> hostValidator; private final PrepareParams params; private final FileRegistry fileRegistry; private final Optional<ApplicationSet> currentActiveApplicationSet; - private final ModelContext.Properties properties; private final Curator curator; private final ExecutorService executor; public PreparedModelsBuilder(ModelFactoryRegistry modelFactoryRegistry, PermanentApplicationPackage permanentApplicationPackage, + FlagSource flagSource, + SecretStore secretStore, + List<ContainerEndpoint> containerEndpoints, + Optional<EndpointCertificateSecrets> endpointCertificateSecrets, ConfigDefinitionRepo configDefinitionRepo, FileRegistry fileRegistry, ExecutorService executor, @@ -71,17 +84,20 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P DeployLogger deployLogger, PrepareParams params, Optional<ApplicationSet> currentActiveApplicationSet, - ModelContext.Properties properties, - ConfigserverConfig configserverConfig) { - super(modelFactoryRegistry, configserverConfig, properties.zone(), hostProvisionerProvider, deployLogger); + ConfigserverConfig configserverConfig, + Zone zone) { + super(modelFactoryRegistry, configserverConfig, zone, hostProvisionerProvider, deployLogger); this.permanentApplicationPackage = permanentApplicationPackage; + this.flagSource = flagSource; + this.secretStore = secretStore; + this.containerEndpoints = containerEndpoints; + this.endpointCertificateSecrets = endpointCertificateSecrets; this.configDefinitionRepo = configDefinitionRepo; this.fileRegistry = fileRegistry; this.hostValidator = hostValidator; this.curator = curator; this.params = params; this.currentActiveApplicationSet = currentActiveApplicationSet; - this.properties = properties; this.executor = executor; } @@ -107,7 +123,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P new ApplicationCuratorDatabase(applicationId.tenant(), curator).readReindexingStatus(applicationId), createHostProvisioner(applicationPackage, provisioned), provisioned, - properties, + createModelContextProperties(modelFactory.version(), applicationPackage), getAppDir(applicationPackage), wantedDockerImageRepository, modelVersion, @@ -118,7 +134,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P } private ModelCreateResult createAndValidateModel(ModelFactory modelFactory, ApplicationId applicationId, Version modelVersion, ModelContext modelContext) { - log.log(properties.zone().system().isCd() ? Level.INFO : Level.FINE, + log.log(zone().system().isCd() ? Level.INFO : Level.FINE, () -> "Create and validate model " + modelVersion + " for " + applicationId + ", previous model is " + modelOf(modelVersion).map(Model::version).map(Version::toFullString).orElse("non-existing")); ValidationParameters validationParameters = @@ -139,7 +155,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P private HostProvisioner createHostProvisioner(ApplicationPackage applicationPackage, Provisioned provisioned) { HostProvisioner defaultHostProvisioner = DeployState.getDefaultModelHostProvisioner(applicationPackage); // Note: nodeRepositoryProvisioner will always be present when hosted is true - Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(properties.applicationId(), provisioned); + Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(params.getApplicationId(), provisioned); Optional<AllocatedHosts> allocatedHosts = applicationPackage.getAllocatedHosts(); if (allocatedHosts.isEmpty()) return nodeRepositoryProvisioner.orElse(defaultHostProvisioner); @@ -184,6 +200,25 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P throw exception; } + private ModelContext.Properties createModelContextProperties(Version modelVersion, + ApplicationPackage applicationPackage) { + return new ModelContextImpl.Properties(params.getApplicationId(), + modelVersion, + configserverConfig, + zone(), + Set.copyOf(containerEndpoints), + params.isBootstrap(), + currentActiveApplicationSet.isEmpty(), + LegacyFlags.from(applicationPackage, flagSource), + endpointCertificateSecrets, + params.athenzDomain(), + params.quota(), + params.tenantSecretStores(), + secretStore, + params.operatorCertificates(), + params.cloudAccount()); + } + /** The result of preparing a single model version */ public static class PreparedModelResult implements ModelResult { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index 881d3de4eaa..edc166d0989 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -115,6 +115,10 @@ public abstract class Session implements Comparable<Session> { return sessionZooKeeperClient.readCreateTime(); } + public Instant getActivatedTime() { + return sessionZooKeeperClient.readActivatedTime(); + } + public void setApplicationId(ApplicationId applicationId) { sessionZooKeeperClient.writeApplicationId(applicationId); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 4c1da9dc3af..1e073ac3458 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -26,6 +26,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.net.HostName; import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; @@ -61,6 +62,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.vespa.config.server.ConfigServerSpec.fromConfig; + /** * A SessionPreparer is responsible for preparing a session given an application package. * @@ -157,7 +160,6 @@ public class SessionPreparer { final ContainerEndpointsCache containerEndpointsCache; final List<ContainerEndpoint> containerEndpoints; - final ModelContext.Properties properties; private final EndpointCertificateMetadataStore endpointCertificateMetadataStore; private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private final Optional<AthenzDomain> athenzDomain; @@ -191,24 +193,13 @@ public class SessionPreparer { .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); this.containerEndpoints = readEndpointsIfNull(params.containerEndpoints()); this.athenzDomain = params.athenzDomain(); - this.properties = new ModelContextImpl.Properties(params.getApplicationId(), - vespaVersion, - configserverConfig, - zone, - Set.copyOf(containerEndpoints), - params.isBootstrap(), - currentActiveApplicationSet.isEmpty(), - LegacyFlags.from(applicationPackage, flagSource), - endpointCertificateSecrets, - athenzDomain, - params.quota(), - params.tenantSecretStores(), - secretStore, - params.operatorCertificates(), - params.cloudAccount()); this.fileRegistry = fileDistributionFactory.createFileRegistry(serverDbSessionDir); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, + flagSource, + secretStore, + containerEndpoints, + endpointCertificateSecrets, configDefinitionRepo, fileRegistry, executor, @@ -218,8 +209,8 @@ public class SessionPreparer { logger, params, currentActiveApplicationSet, - properties, - configserverConfig); + configserverConfig, + zone); } void checkTimeout(String step) { @@ -236,7 +227,7 @@ public class SessionPreparer { FileDistribution fileDistribution = fileDistributionFactory.createFileDistribution(); log.log(Level.FINE, () -> "Ask other config servers to download application package for " + applicationId + " (" + fileReference + ")"); - properties.configServerSpecs() + ConfigServerSpec.fromConfig(configserverConfig) .stream() .filter(spec -> !spec.getHostName().equals(HostName.getLocalhost())) .forEach(spec -> fileDistribution.startDownload(spec.getHostName(), spec.getConfigServerPort(), Set.of(fileReference))); @@ -247,7 +238,7 @@ public class SessionPreparer { void preprocess() { try { - this.preprocessedApplicationPackage = applicationPackage.preprocess(properties.zone(), logger); + this.preprocessedApplicationPackage = applicationPackage.preprocess(zone, logger); } catch (IOException | RuntimeException e) { throw new IllegalArgumentException("Error preprocessing application package for " + applicationId + ", session " + sessionZooKeeperClient.sessionId(), e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 5603ef2df51..059d192e7d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -235,7 +235,7 @@ public class SessionRepository { public ConfigChangeActions prepareLocalSession(Session session, DeployLogger logger, PrepareParams params, Instant now) { params.vespaVersion().ifPresent(version -> { if ( ! params.isBootstrap() && ! modelFactoryRegistry.allVersions().contains(version)) - throw new UnknownVespaVersionException("Vespa version '" + version + "' not known by this configserver"); + throw new UnknownVespaVersionException("Vespa version '" + version + "' not known by this config server"); }); applicationRepo.createApplication(params.getApplicationId()); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index de8cbcb4066..e146f0de187 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -33,7 +33,7 @@ import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; - +import org.apache.zookeeper.data.Stat; import java.security.cert.X509Certificate; import java.time.Instant; import java.util.List; @@ -239,6 +239,11 @@ public class SessionZooKeeperClient { return data.map(d -> Instant.ofEpochSecond(Long.parseLong(Utf8.toString(d)))).orElse(Instant.EPOCH); } + public Instant readActivatedTime() { + Optional<Stat> statData = curator.getStat(sessionStatusPath); + return statData.map(s -> Instant.ofEpochMilli(s.getMtime())).orElse(Instant.EPOCH); + } + private Path getCreateTimePath() { return sessionPath.append(CREATE_TIME_PATH); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index d9ca31d561d..ee2a979be7a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -7,7 +7,6 @@ import com.yahoo.component.Version; import com.yahoo.config.ConfigInstance; import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.application.api.ApplicationMetaData; -import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; @@ -57,7 +56,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; - +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; @@ -75,6 +74,7 @@ import java.util.Map; import java.util.Optional; import java.util.stream.IntStream; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -245,10 +245,13 @@ public class ApplicationRepositoryTest { } @Test - public void getLogs() { + public void getLogs() throws IOException { deployApp(testAppLogServerWithContainer); HttpResponse response = applicationRepository.getLogs(applicationId(), Optional.empty(), ""); assertEquals(200, response.getStatus()); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + response.render(buffer); + assertEquals("log line", buffer.toString(UTF_8)); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java index 6a95c059fcc..4a521e3b8a2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java @@ -7,6 +7,8 @@ import com.yahoo.vespa.config.server.http.LogRetriever; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Optional; /** * @author olaa @@ -14,21 +16,14 @@ import java.nio.charset.StandardCharsets; public class MockLogRetriever extends LogRetriever { @Override - public HttpResponse getLogs(String logServerUri) { - return new MockHttpResponse(); - } - - private static class MockHttpResponse extends HttpResponse { - - private MockHttpResponse() { - super(200); - } - - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write("log line".getBytes(StandardCharsets.UTF_8)); - } - + public HttpResponse getLogs(String logServerUri, Optional<Instant>deployTime ) { + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("log line".getBytes(StandardCharsets.UTF_8)); + } + + }; } }
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index 2d60dc3f37b..93881b52eb6 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -43,8 +43,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; * @author jonmv */ class LogReader { - static final Pattern logArchivePathPattern = Pattern.compile("(\\d{4})/(\\d{2})/(\\d{2})/(\\d{2})-\\d+(.gz)?"); - static final Pattern vespaLogPathPattern = Pattern.compile("vespa\\.log(?:-(\\d{4})-(\\d{2})-(\\d{2})\\.(\\d{2})-(\\d{2})-(\\d{2})(?:.gz)?)?"); + static final Pattern logArchivePathPattern = Pattern.compile("(\\d{4})/(\\d{2})/(\\d{2})/(\\d{2})-\\d+(\\.gz|\\.zst)?"); + static final Pattern vespaLogPathPattern = Pattern.compile("vespa\\.log(?:-(\\d{4})-(\\d{2})-(\\d{2})\\.(\\d{2})-(\\d{2})-(\\d{2})(?:\\.gz|\\.zst)?)?"); private final Path logDirectory; private final Pattern logFilePattern; @@ -97,23 +97,44 @@ class LogReader { private final double to; private final Optional<String> hostname; private LineWithTimestamp next; + private Process zcat = null; - private LogLineIterator(Path log, double from, double to, Optional<String> hostname) throws IOException { - boolean zipped = log.toString().endsWith(".gz"); - InputStream in = InputStream.nullInputStream(); + private InputStream openFile(Path log) { + boolean gzipped = log.toString().endsWith(".gz"); + boolean is_zstd = log.toString().endsWith(".zst"); try { - in = Files.newInputStream(log); - } - catch (NoSuchFileException e) { // File may have been compressed since we found it. - if ( ! zipped) + if (gzipped) { + var in_gz = Files.newInputStream(log); + return new GZIPInputStream(in_gz); + } else if (is_zstd) { + var pb = new ProcessBuilder("zstdcat", log.toString()); + pb.redirectError(ProcessBuilder.Redirect.DISCARD); + zcat = pb.start(); + zcat.getOutputStream().close(); + return zcat.getInputStream(); + } else { try { - in = Files.newInputStream(Paths.get(log + ".gz")); - zipped = true; + return Files.newInputStream(log); + } catch (NoSuchFileException e) { // File may have been compressed since we found it. + Path p = Paths.get(log + ".gz"); + if (Files.exists(p)) { + return openFile(p); + } + p = Paths.get(log + ".zst"); + if (Files.exists(p)) { + return openFile(p); + } } - catch (NoSuchFileException ignored) { } + } + } catch (IOException ignored) { } + // failure fallback: + return InputStream.nullInputStream(); + } - this.reader = new BufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(in) : in, UTF_8)); + private LogLineIterator(Path log, double from, double to, Optional<String> hostname) throws IOException { + InputStream in = openFile(log); + this.reader = new BufferedReader(new InputStreamReader(in, UTF_8)); this.from = from; this.to = to; this.hostname = hostname; @@ -135,6 +156,9 @@ class LogReader { @Override public void close() throws IOException { reader.close(); + if (zcat != null) { + zcat.destroy(); + } } private LineWithTimestamp readNext() { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java index 927284f2c66..cded4f36d54 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc; -import com.google.inject.Key; import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.jdisc.Container; import com.yahoo.jdisc.References; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index d4cf6324721..1f95e7fc75e 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -27,7 +27,7 @@ import java.util.logging.Logger; /** * A request handler which assigns a worker thread to handle each request. - * This is mean to be subclasses by handlers who does work by executing each + * This is meant to be subclassed by handlers who do work by executing each * request in a separate thread. * <p> * Note that this means that subclass handlers are synchronous - the request io can diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java index 97ea30b3867..ae2871e266d 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.Executor; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -182,6 +183,8 @@ class HealthCheckProxyHandler extends HandlerWrapper { private StatusResponse getStatusResponse() { try { var request = client().newRequest("https://localhost:" + port + HEALTH_CHECK_PATH); + request.timeout(timeout.toMillis(), TimeUnit.MILLISECONDS); + request.idleTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS); if (proxyProtocol) { request.tag(new ProxyProtocolClientConnectionFactory.V1.Tag()); } @@ -204,18 +207,18 @@ class HealthCheckProxyHandler extends HandlerWrapper { synchronized (this) { if (client == null) { int timeoutMillis = (int) timeout.toMillis(); - SslContextFactory.Client clientSsl = new SslContextFactory.Client(); - clientSsl.setHostnameVerifier((__, ___) -> true); - clientSsl.setSslContext(getSslContext(serverSsl)); - HttpClient client = new HttpClient(clientSsl); - client.setMaxConnectionsPerDestination(4); - client.setConnectTimeout(timeoutMillis); - client.setStopTimeout(timeoutMillis); - client.setIdleTimeout(timeoutMillis); - client.setUserAgentField(new HttpField(HttpHeader.USER_AGENT, "health-check-proxy-client")); - client.start(); - this.client = client; - } + SslContextFactory.Client clientSsl = new SslContextFactory.Client(); + clientSsl.setHostnameVerifier((__, ___) -> true); + clientSsl.setSslContext(getSslContext(serverSsl)); + HttpClient client = new HttpClient(clientSsl); + client.setMaxConnectionsPerDestination(4); + client.setConnectTimeout(timeoutMillis); + client.setStopTimeout(timeoutMillis); + client.setIdleTimeout(timeoutMillis); + client.setUserAgentField(new HttpField(HttpHeader.USER_AGENT, "health-check-proxy-client")); + client.start(); + this.client = client; + } } } return client; diff --git a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java index efed58f4ab0..5e52f8d8b37 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java +++ b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.request; -import com.google.common.collect.ImmutableList; - +import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -25,7 +24,7 @@ public final class CompoundName { private final String name; private final String lowerCasedName; - private final ImmutableList<String> compounds; + private final List<String> compounds; /** A hashcode which is always derived from the compounds (NEVER the string) */ private final int hashCode; @@ -42,16 +41,20 @@ public final class CompoundName { * @throws NullPointerException if name is null */ public CompoundName(String name) { - this(name, parse(name)); + this(name, parse(name).toArray(new String[1])); } /** Constructs this from an array of name components which are assumed not to contain dots */ public static CompoundName fromComponents(String ... components) { - return new CompoundName(Arrays.asList(components)); + return new CompoundName(List.of(components)); } /** Constructs this from a list of compounds. */ public CompoundName(List<String> compounds) { + this(compounds.toArray(new String[compounds.size()])); + } + + private CompoundName(String [] compounds) { this(toCompoundString(compounds), compounds); } @@ -62,25 +65,26 @@ public final class CompoundName { * @param name the string representation of the compounds * @param compounds the compounds of this name */ - private CompoundName(String name, List<String> compounds) { + private CompoundName(String name, String [] compounds) { if (name == null) throw new NullPointerException("Name can not be null"); this.name = name; this.lowerCasedName = toLowerCase(name); - if (compounds.size() == 1 && compounds.get(0).isEmpty()) - this.compounds = ImmutableList.of(); - else - this.compounds = ImmutableList.copyOf(compounds); + if (compounds.length == 1 && compounds[0].isEmpty()) { + this.compounds = List.of(); + this.hashCode = 0; + rest = this; + return; + } + this.compounds = new ImmutableArrayList(compounds); this.hashCode = this.compounds.hashCode(); - int size = this.compounds.size(); - rest = size > 1 ? new CompoundName(compounds.subList(1, size)) - : size == 1 ? empty : this; // size==0 -> this needed during construction of empty + rest = compounds.length > 1 ? new CompoundName(name.substring(compounds[0].length()+1), Arrays.copyOfRange(compounds, 1, compounds.length)) + : empty; } private static List<String> parse(String s) { ArrayList<String> l = null; - int p = 0; final int m = s.length(); for (int i = 0; i < m; i++) { @@ -91,7 +95,8 @@ public final class CompoundName { } } if (p == 0) { - return ImmutableList.of(s); + if (l == null) return List.of(s); + l.add(s); } else if (p < m) { l.add(s.substring(p, m)); } else { @@ -107,10 +112,7 @@ public final class CompoundName { */ public CompoundName append(String name) { if (name.isEmpty()) return this; - if (isEmpty()) return new CompoundName(name); - List<String> newCompounds = new ArrayList<>(compounds); - newCompounds.addAll(parse(name)); - return new CompoundName(concat(this.name, name), newCompounds); + return append(new CompoundName(name)); } /** @@ -121,12 +123,14 @@ public final class CompoundName { public CompoundName append(CompoundName name) { if (name.isEmpty()) return this; if (isEmpty()) return name; - List<String> newCompounds = new ArrayList<>(compounds); - newCompounds.addAll(name.compounds); + String [] newCompounds = new String[compounds.size() + name.compounds.size()]; + int count = 0; + for (String s : compounds) { newCompounds[count++] = s; } + for (String s : name.compounds) { newCompounds[count++] = s; } return new CompoundName(concat(this.name, name.name), newCompounds); } - private String concat(String name1, String name2) { + private static String concat(String name1, String name2) { return name1 + "." + name2; } @@ -215,7 +219,7 @@ public final class CompoundName { * As an optimization, if the given name == the name component at this index, this is returned. */ public CompoundName set(int i, String name) { - if (get(i) == name) return this; + if (get(i).equals(name)) return this; List<String> newCompounds = new ArrayList<>(compounds); newCompounds.set(i, name); return new CompoundName(newCompounds); @@ -261,11 +265,9 @@ public final class CompoundName { public int hashCode() { return hashCode; } @Override - public boolean equals(Object o) { - if (o == this) return true; - if ( ! (o instanceof CompoundName)) return false; - CompoundName other = (CompoundName)o; - return this.name.equals(other.name); + public boolean equals(Object arg) { + if (arg == this) return true; + return (arg instanceof CompoundName o) && name.equals(o.name); } /** @@ -278,13 +280,42 @@ public final class CompoundName { return lowerCasedName; } - private static String toCompoundString(List<String> compounds) { - StringBuilder b = new StringBuilder(); - for (String compound : compounds) - b.append(compound).append("."); + private static String toCompoundString(String [] compounds) { + int all = compounds.length; + for (int i = 0; i < compounds.length; i++) + all += compounds[i].length(); + StringBuilder b = new StringBuilder(all); + for (int i = 0; i < compounds.length; i++) + b.append(compounds[i]).append("."); return b.length()==0 ? "" : b.substring(0, b.length()-1); } public static CompoundName from(String name) { return new CompoundName(name); } + private static class ImmutableArrayList extends AbstractList<String> { + + private final String [] array; + ImmutableArrayList(String [] array) { + this.array = array; + } + @Override + public String get(int index) { + return array[index]; + } + + @Override + public int size() { + return array.length; + } + + @Override + public int hashCode() { + int hashCode = 0; + for (String s : array) { + hashCode = hashCode ^ s.hashCode(); + } + return hashCode; + } + } + } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index 120ee49031d..e1bd0da457e 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -1,9 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; -import com.yahoo.vespa.test.file.TestFileSystem; +import com.yahoo.compress.ZstdCompressor; import org.junit.Before; import org.junit.Test; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -22,8 +24,9 @@ import static org.junit.Assert.assertEquals; public class LogReaderTest { - private final FileSystem fileSystem = TestFileSystem.create(); - private final Path logDirectory = fileSystem.getPath("/opt/vespa/logs"); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + private Path logDirectory; private static final String logv11 = "3600.2\tnode1.com\t5480\tcontainer\tstdout\tinfo\tfourth\n"; private static final String logv = "90000.1\tnode1.com\t5480\tcontainer\tstdout\tinfo\tlast\n"; @@ -34,12 +37,13 @@ public class LogReaderTest { @Before public void setup() throws IOException { + logDirectory = folder.newFolder("opt/vespa/logs").toPath(); // Log archive paths and file names indicate what hour they contain logs for, with the start of that hour. // Multiple entries may exist for each hour. Files.createDirectories(logDirectory.resolve("1970/01/01")); - Files.write(logDirectory.resolve("1970/01/01/00-0.gz"), compress(log100)); + Files.write(logDirectory.resolve("1970/01/01/00-0.gz"), compress1(log100)); Files.write(logDirectory.resolve("1970/01/01/00-1"), log101.getBytes(UTF_8)); - Files.write(logDirectory.resolve("1970/01/01/01-0.gz"), compress(log110)); + Files.write(logDirectory.resolve("1970/01/01/01-0.zst"), compress2(log110)); Files.createDirectories(logDirectory.resolve("1970/01/02")); Files.write(logDirectory.resolve("1970/01/02/00-0"), log200.getBytes(UTF_8)); @@ -86,7 +90,7 @@ public class LogReaderTest { assertEquals(log101 + log100 + log200, baos.toString(UTF_8)); } - private byte[] compress(String input) throws IOException { + private byte[] compress1(String input) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream zip = new GZIPOutputStream(baos); zip.write(input.getBytes()); @@ -94,4 +98,9 @@ public class LogReaderTest { return baos.toByteArray(); } + private byte[] compress2(String input) throws IOException { + byte[] data = input.getBytes(); + return new ZstdCompressor().compress(data, 0, data.length); + } + } diff --git a/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java b/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java index 841866f1e83..a8a2c76a609 100644 --- a/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java +++ b/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java @@ -1,17 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.request; - -import static org.junit.Assert.*; +import com.google.common.base.Splitter; +import com.yahoo.text.Lowercase; import java.util.Iterator; import java.util.List; -import org.junit.After; -import org.junit.Before; import org.junit.Test; -import com.google.common.base.Splitter; -import com.yahoo.text.Lowercase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; /** * Module local test of the basic property name building block. @@ -21,17 +21,15 @@ import com.yahoo.text.Lowercase; public class CompoundNameTestCase { private static final String NAME = "com.yahoo.processing.request.CompoundNameTestCase"; - private CompoundName cn; + private final CompoundName cn = new CompoundName(NAME); - @Before - public void setUp() throws Exception { - cn = new CompoundName(NAME); + void verifyStrict(CompoundName expected, CompoundName actual) { + assertEquals(expected, actual); + assertEquals(expected.asList(), actual.asList()); } - - @After - public void tearDown() throws Exception { + void verifyStrict(String expected, CompoundName actual) { + verifyStrict(new CompoundName(expected), actual); } - @Test public final void testLast() { assertEquals(NAME.substring(NAME.lastIndexOf('.') + 1), cn.last()); @@ -44,17 +42,17 @@ public class CompoundNameTestCase { @Test public final void testRest() { - assertEquals(NAME.substring(NAME.indexOf('.') + 1), cn.rest().toString()); + verifyStrict(NAME.substring(NAME.indexOf('.') + 1), cn.rest()); } @Test public final void testRestN() { - assertEquals("a.b.c.d.e", new CompoundName("a.b.c.d.e").rest(0).toString()); - assertEquals("b.c.d.e", new CompoundName("a.b.c.d.e").rest(1).toString()); - assertEquals("c.d.e", new CompoundName("a.b.c.d.e").rest(2).toString()); - assertEquals("d.e", new CompoundName("a.b.c.d.e").rest(3).toString()); - assertEquals("e", new CompoundName("a.b.c.d.e").rest(4).toString()); - assertEquals("", new CompoundName("a.b.c.d.e").rest(5).toString()); + verifyStrict("a.b.c.d.e", new CompoundName("a.b.c.d.e").rest(0)); + verifyStrict("b.c.d.e", new CompoundName("a.b.c.d.e").rest(1)); + verifyStrict("c.d.e", new CompoundName("a.b.c.d.e").rest(2)); + verifyStrict("d.e", new CompoundName("a.b.c.d.e").rest(3)); + verifyStrict("e", new CompoundName("a.b.c.d.e").rest(4)); + verifyStrict(CompoundName.empty, new CompoundName("a.b.c.d.e").rest(5)); } @Test @@ -69,6 +67,12 @@ public class CompoundNameTestCase { } @Test + public void testFromComponents() { + verifyStrict("a", CompoundName.fromComponents("a")); + verifyStrict("a.b", CompoundName.fromComponents("a", "b")); + } + + @Test public final void testSize() { Splitter s = Splitter.on('.'); Iterable<String> i = s.split(NAME); @@ -111,8 +115,8 @@ public class CompoundNameTestCase { public final void testEqualsObject() { assertFalse(cn.equals(NAME)); assertFalse(cn.equals(null)); - assertTrue(cn.equals(cn)); - assertTrue(cn.equals(new CompoundName(NAME))); + verifyStrict(cn, cn); + verifyStrict(cn, new CompoundName(NAME)); } @Test @@ -122,8 +126,8 @@ public class CompoundNameTestCase { assertFalse(new CompoundName("a").isEmpty()); assertEquals(1, new CompoundName("a").size()); CompoundName empty = new CompoundName("a.b.c"); - assertTrue(empty == empty.rest(0)); - assertFalse(empty == empty.rest(1)); + verifyStrict(empty, empty.rest(0)); + assertNotEquals(empty, empty.rest(1)); } @Test @@ -132,12 +136,12 @@ public class CompoundNameTestCase { } @Test - public void testAppend() { - assertEquals(new CompoundName("a.b.c.d"), new CompoundName("").append(new CompoundName("a.b.c.d"))); - assertEquals(new CompoundName("a.b.c.d"), new CompoundName("a").append(new CompoundName("b.c.d"))); - assertEquals(new CompoundName("a.b.c.d"), new CompoundName("a.b").append(new CompoundName("c.d"))); - assertEquals(new CompoundName("a.b.c.d"), new CompoundName("a.b.c").append(new CompoundName("d"))); - assertEquals(new CompoundName("a.b.c.d"), new CompoundName("a.b.c.d").append(new CompoundName(""))); + public void testAppendCompound() { + verifyStrict("a.b.c.d", new CompoundName("").append(new CompoundName("a.b.c.d"))); + verifyStrict("a.b.c.d", new CompoundName("a").append(new CompoundName("b.c.d"))); + verifyStrict("a.b.c.d", new CompoundName("a.b").append(new CompoundName("c.d"))); + verifyStrict("a.b.c.d", new CompoundName("a.b.c").append(new CompoundName("d"))); + verifyStrict("a.b.c.d", new CompoundName("a.b.c.d").append(new CompoundName(""))); } @Test @@ -155,4 +159,59 @@ public class CompoundNameTestCase { assertFalse(name.hasPrefix(stringPrefix)); } + + @Test + public void testFirstRest() { + verifyStrict(CompoundName.empty, CompoundName.empty.rest()); + + CompoundName n=new CompoundName("on.two.three"); + assertEquals("on", n.first()); + verifyStrict("two.three", n.rest()); + n=n.rest(); + assertEquals("two", n.first()); + verifyStrict("three", n.rest()); + n=n.rest(); + assertEquals("three", n.first()); + verifyStrict("", n.rest()); + n=n.rest(); + assertEquals("", n.first()); + verifyStrict("", n.rest()); + n=n.rest(); + assertEquals("", n.first()); + verifyStrict("", n.rest()); + } + + @Test + public void testHashCodeAndEquals() { + CompoundName n1 = new CompoundName("venn.d.a"); + CompoundName n2 = new CompoundName(n1.asList()); + assertEquals(n1.hashCode(), n2.hashCode()); + verifyStrict(n1, n2); + } + + @Test + public void testAppendString() { + verifyStrict("a", new CompoundName("a").append("")); + verifyStrict("a", new CompoundName("").append("a")); + verifyStrict("a.b", new CompoundName("a").append("b")); + verifyStrict("a.b.c.d", new CompoundName("a.b").append("c.d")); + + CompoundName name = new CompoundName("a.b"); + verifyStrict("a.b.c", name.append("c")); + verifyStrict("a.b.d", name.append("d")); + verifyStrict("a.b.d.e", name.append("d.e")); + } + + @Test + public void testEmpty() { + CompoundName empty=new CompoundName(""); + assertEquals("", empty.toString()); + assertEquals(0, empty.asList().size()); + } + + @Test + public void testAsList2() { + assertEquals("[one]", new CompoundName("one").asList().toString()); + assertEquals("[one, two, three]", new CompoundName("one.two.three").asList().toString()); + } } diff --git a/container-core/src/test/java/com/yahoo/processing/request/test/CompoundNameBenchmark.java b/container-core/src/test/java/com/yahoo/processing/request/test/CompoundNameBenchmark.java index 81ddc42f6c3..144c3cf736e 100644 --- a/container-core/src/test/java/com/yahoo/processing/request/test/CompoundNameBenchmark.java +++ b/container-core/src/test/java/com/yahoo/processing/request/test/CompoundNameBenchmark.java @@ -9,34 +9,37 @@ import com.yahoo.processing.request.CompoundName; public class CompoundNameBenchmark { public void run() { long result=0; - String strings[] = createStrings(1000); + String [] strings = createStrings(1000); // Warm-up out("Warming up..."); - for (int i=0; i<10*1000; i++) + for (int i=0; i<2*1000; i++) result+=createCompundName(strings); long startTime=System.currentTimeMillis(); out("Running..."); - for (int i=0; i<100*1000; i++) + for (int i=0; i<10*1000; i++) result+=createCompundName(strings); out("Ignore this: " + result); // Make sure we are not fooled by optimization by creating an observable result long endTime=System.currentTimeMillis(); out("Compoundification 1000 strings 100.000 times took " + (endTime-startTime) + " ms"); } - private final String [] createStrings(int num) { - String strings [] = new String [num]; + private String [] createStrings(int num) { + String [] strings = new String [num]; for(int i=0; i < strings.length; i++) { strings[i] = "this.is.a.short.compound.name." + i; } return strings; } - private final int createCompundName(String [] strings) { + private int createCompundName(String [] strings) { int retval = 0; - for (int i=0; i < strings.length; i++) { - CompoundName n = new CompoundName(strings[i]); + CompoundName toAppend = new CompoundName("appended.twice"); + for (String s : strings) { + CompoundName n = new CompoundName(s); retval += n.size(); + CompoundName a = n.append(toAppend); + retval += a.size(); } return retval; } diff --git a/container-core/src/test/java/com/yahoo/processing/request/test/CompoundNameTestCase.java b/container-core/src/test/java/com/yahoo/processing/request/test/CompoundNameTestCase.java deleted file mode 100644 index df9e77938a6..00000000000 --- a/container-core/src/test/java/com/yahoo/processing/request/test/CompoundNameTestCase.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.processing.request.test; - -import com.yahoo.processing.request.CompoundName; -import static org.junit.Assert.assertEquals; -import org.junit.Test; - -/** - * @author bratseth - */ -public class CompoundNameTestCase { - - @Test - public void testFirstRest() { - assertEquals(CompoundName.empty, CompoundName.empty.rest()); - - CompoundName n=new CompoundName("on.two.three"); - assertEquals("on", n.first()); - assertEquals("two.three", n.rest().toString()); - n=n.rest(); - assertEquals("two", n.first()); - assertEquals("three", n.rest().toString()); - n=n.rest(); - assertEquals("three", n.first()); - assertEquals("", n.rest().toString()); - n=n.rest(); - assertEquals("", n.first()); - assertEquals("", n.rest().toString()); - n=n.rest(); - assertEquals("", n.first()); - assertEquals("", n.rest().toString()); - } - - @Test - public void testHashCodeAndEquals() { - CompoundName n1 = new CompoundName("venn.d.a"); - CompoundName n2 = new CompoundName(n1.asList()); - assertEquals(n1.hashCode(), n2.hashCode()); - assertEquals(n1, n2); - } - - @Test - public void testAppend() { - assertEquals("a",new CompoundName("a").append("").toString()); - assertEquals("a",new CompoundName("").append("a").toString()); - assertEquals("a.b",new CompoundName("a").append("b").toString()); - - CompoundName name = new CompoundName("a.b"); - assertEquals("a.b.c",name.append("c").toString()); - assertEquals("a.b.d",name.append("d").toString()); - } - - @Test - public void testEmpty() { - CompoundName empty=new CompoundName(""); - assertEquals("", empty.toString()); - assertEquals(0, empty.asList().size()); - } - - @Test - public void testAsList() { - assertEquals("[one]", new CompoundName("one").asList().toString()); - assertEquals("[one, two, three]", new CompoundName("one.two.three").asList().toString()); - } - -} diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index b8d661de111..a88b7e1d577 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -19,15 +19,16 @@ <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>container</artifactId> + <artifactId>container-test</artifactId> <version>${project.version}</version> - <scope>provided</scope> + <scope>test</scope> </dependency> + <!-- Declare container-test before container here, opposite of what we do in hosted-tenant-base, to cover both cases --> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>container-test</artifactId> + <artifactId>container</artifactId> <version>${project.version}</version> - <scope>test</scope> + <scope>provided</scope> </dependency> <dependency> <groupId>junit</groupId> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 715340c4913..8279a497e4a 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -17,13 +17,6 @@ <packaging>jar</packaging> <dependencies> <dependency> - <!-- This dep is a workaround for bugs 7272469, 7259035 --> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - <version>${jackson2.version}</version> - </dependency> - - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>jdisc_core</artifactId> <version>${project.version}</version> @@ -159,10 +152,6 @@ <artifactId>commons-exec</artifactId> </exclusion> <exclusion> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </exclusion> - <exclusion> <groupId>net.java.dev.jna</groupId> <artifactId>jna</artifactId> </exclusion> @@ -248,12 +237,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>jrt</artifactId> <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>org.bouncycastle</groupId> - <artifactId>bcpkix-jdk15on</artifactId> - </exclusion> - </exclusions> </dependency> <!-- NOTE: Dependencies below are added explicitly to exclude transitive deps that are not provided runtime by the container, diff --git a/container-disc/abi-spec.json b/container-disc/abi-spec.json index aae37552283..968cef8e88f 100644 --- a/container-disc/abi-spec.json +++ b/container-disc/abi-spec.json @@ -17,6 +17,7 @@ "public abstract java.lang.String getAccessToken(java.lang.String)", "public abstract java.lang.String getAccessToken(java.lang.String, java.util.List)", "public abstract java.util.List getIdentityCertificate()", + "public abstract java.security.cert.X509Certificate getRoleCertificate(java.lang.String, java.lang.String)", "public abstract java.security.PrivateKey getPrivateKey()", "public abstract java.nio.file.Path trustStorePath()" ], diff --git a/container-disc/pom.xml b/container-disc/pom.xml index 95f54e5d932..3e42b924ec6 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -188,13 +188,8 @@ <!-- Vespa bundles --> config-bundle-jar-with-dependencies.jar, configdefinitions-jar-with-dependencies.jar, - container-search-and-docproc-jar-with-dependencies.jar, - container-search-gui-jar-with-dependencies.jar, - docprocs-jar-with-dependencies.jar, hosted-zone-api-jar-with-dependencies.jar, jdisc-security-filters-jar-with-dependencies.jar, - linguistics-components-jar-with-dependencies.jar, - vespaclient-container-plugin-jar-with-dependencies.jar, vespa-athenz-jar-with-dependencies.jar, <!-- Apache http client repackaged as bundle --> diff --git a/container-search-and-docproc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java b/container-disc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java index 09b5a6ff85e..67862533259 100644 --- a/container-search-and-docproc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java +++ b/container-disc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java @@ -6,18 +6,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.yahoo.component.annotation.Inject; -import com.yahoo.component.AbstractComponent; +import com.yahoo.component.Component; import com.yahoo.component.ComponentId; import com.yahoo.component.Vtag; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.chain.Chain; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.Container; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; -import com.yahoo.docproc.Call; -import com.yahoo.docproc.impl.DocprocService; -import com.yahoo.docproc.jdisc.DocumentProcessingHandler; import com.yahoo.jdisc.handler.AbstractRequestHandler; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; @@ -32,15 +29,13 @@ import com.yahoo.jdisc.service.ServerProvider; import com.yahoo.processing.Processor; import com.yahoo.processing.execution.chain.ChainRegistry; import com.yahoo.processing.handler.ProcessingHandler; -import com.yahoo.search.handler.SearchHandler; -import com.yahoo.search.searchchain.SearchChainRegistry; import org.osgi.framework.Bundle; import org.osgi.framework.FrameworkUtil; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -49,9 +44,14 @@ import java.util.Map; * * @author gjoranv * @author Einar M R Rosenvinge + * @author bjorncs */ public class ApplicationStatusHandler extends AbstractRequestHandler { + public interface Extension { + Map<String, ? extends JsonNode> produceExtraFields(ApplicationStatusHandler handler); + } + private static final ObjectMapper jsonMapper = new ObjectMapper(); private final JsonNode applicationJson; @@ -60,6 +60,7 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { private final JsonNode requestFiltersJson; private final JsonNode responseFiltersJson; private final JdiscBindingsConfig bindingsConfig; + private final Collection<Extension> extensions; @Inject public ApplicationStatusHandler(ApplicationMetadataConfig metaConfig, @@ -68,8 +69,8 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { ComponentRegistry<ClientProvider> clientProviderRegistry, ComponentRegistry<ServerProvider> serverProviderRegistry, ComponentRegistry<RequestFilterBase> requestFilterRegistry, - ComponentRegistry<ResponseFilterBase> responseFilterRegistry) { - + ComponentRegistry<ResponseFilterBase> responseFilterRegistry, + ComponentRegistry<Extension> extensions) { applicationJson = renderApplicationConfigs(metaConfig, userConfig); clientsJson = renderRequestHandlers(bindingsConfig, clientProviderRegistry.allComponentsById()); serversJson = renderObjectComponents(serverProviderRegistry.allComponentsById()); @@ -77,14 +78,11 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { responseFiltersJson = renderObjectComponents(responseFilterRegistry.allComponentsById()); this.bindingsConfig = bindingsConfig; + this.extensions = extensions.allComponents(); } @Override public ContentChannel handleRequest(com.yahoo.jdisc.Request request, ResponseHandler handler) { - JsonNode json = new StatusResponse(applicationJson, clientsJson, serversJson, - requestFiltersJson, responseFiltersJson, bindingsConfig) - .render(); - FastContentWriter writer = new FastContentWriter(new ResponseDispatch() { @Override protected com.yahoo.jdisc.Response newResponse() { @@ -95,7 +93,7 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { }.connect(handler)); try { - writer.write(jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(json)); + writer.write(jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(render())); } catch (JsonProcessingException e) { throw new RuntimeException("Invalid JSON: " + e.getMessage(), e); } @@ -104,8 +102,18 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { return new IgnoredContent(); } + public ObjectMapper jsonMapper() { return jsonMapper; } + + public Collection<RequestHandler> requestHandlers() { return container().getRequestHandlerRegistry().allComponents(); } + + private Map<ComponentId, RequestHandler> requestHandlersById() { return container().getRequestHandlerRegistry().allComponentsById(); } + + private List<? extends Component> components() { return container().getComponentRegistry().allComponents(); } + + private static Container container() { return Container.get(); } + static JsonNode renderApplicationConfigs(ApplicationMetadataConfig metaConfig, - ApplicationUserdataConfig userConfig) { + ApplicationUserdataConfig userConfig) { ObjectNode vespa = jsonMapper.createObjectNode(); vespa.put("version", Vtag.currentVersion.toString()); @@ -139,7 +147,7 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { } static JsonNode renderRequestHandlers(JdiscBindingsConfig bindingsConfig, - Map<ComponentId, ? extends RequestHandler> handlersById) { + Map<ComponentId, ? extends RequestHandler> handlersById) { ArrayNode ret = jsonMapper.createArrayNode(); for (Map.Entry<ComponentId, ? extends RequestHandler> handlerEntry : handlersById.entrySet()) { @@ -175,17 +183,17 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { return ret; } - private static JsonNode renderAbstractComponents(List<? extends AbstractComponent> components) { + private static JsonNode renderAbstractComponents(List<? extends Component> components) { ArrayNode ret = jsonMapper.createArrayNode(); - for (AbstractComponent c : components) { + for (Component c : components) { JsonNode jc = renderComponent(c, c.getId()); ret.add(jc); } return ret; } - private static ObjectNode renderComponent(Object component, ComponentId id) { + public static ObjectNode renderComponent(Object component, ComponentId id) { ObjectNode jc = jsonMapper.createObjectNode(); jc.put("id", id.stringValue()); addBundleInfo(jc, component); @@ -221,102 +229,48 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { } } - static final class StatusResponse { - private final JsonNode applicationJson; - private final JsonNode clientsJson; - private final JsonNode serversJson; - private final JsonNode requestFiltersJson; - private final JsonNode responseFiltersJson; - private final JdiscBindingsConfig bindingsConfig; - - StatusResponse(JsonNode applicationJson, - JsonNode clientsJson, - JsonNode serversJson, - JsonNode requestFiltersJson, - JsonNode responseFiltersJson, - JdiscBindingsConfig bindingsConfig) { - this.applicationJson = applicationJson; - this.clientsJson = clientsJson; - this.serversJson = serversJson; - this.requestFiltersJson = requestFiltersJson; - this.responseFiltersJson = responseFiltersJson; - this.bindingsConfig = bindingsConfig; - } - - public JsonNode render() { - ObjectNode root = jsonMapper.createObjectNode(); - - root.set("application", applicationJson); - root.set("abstractComponents", - renderAbstractComponents(Container.get().getComponentRegistry().allComponents())); + JsonNode render() { + ObjectNode root = jsonMapper.createObjectNode(); - root.set("handlers", - renderRequestHandlers(bindingsConfig, Container.get().getRequestHandlerRegistry().allComponentsById())); - root.set("clients", clientsJson); - root.set("servers", serversJson); - root.set("httpRequestFilters", requestFiltersJson); - root.set("httpResponseFilters", responseFiltersJson); + root.set("application", applicationJson); + root.set("abstractComponents", + renderAbstractComponents(components())); - root.set("searchChains", renderSearchChains(Container.get())); - root.set("docprocChains", renderDocprocChains(Container.get())); - root.set("processingChains", renderProcessingChains(Container.get())); + root.set("handlers", renderRequestHandlers(bindingsConfig, requestHandlersById())); + root.set("clients", clientsJson); + root.set("servers", serversJson); + root.set("httpRequestFilters", requestFiltersJson); + root.set("httpResponseFilters", responseFiltersJson); - return root; - } - - private static JsonNode renderSearchChains(Container container) { - for (RequestHandler h : container.getRequestHandlerRegistry().allComponents()) { - if (h instanceof SearchHandler) { - SearchChainRegistry scReg = ((SearchHandler) h).getSearchChainRegistry(); - return renderChains(scReg); - } - } - return jsonMapper.createObjectNode(); - } - - private static JsonNode renderDocprocChains(Container container) { - ObjectNode ret = jsonMapper.createObjectNode(); - for (RequestHandler h : container.getRequestHandlerRegistry().allComponents()) { - if (h instanceof DocumentProcessingHandler) { - ComponentRegistry<DocprocService> registry = ((DocumentProcessingHandler) h).getDocprocServiceRegistry(); - for (DocprocService service : registry.allComponents()) { - ret.set(service.getId().stringValue(), renderCalls(service.getCallStack().iterator())); - } - } - } - return ret; - } + root.set("processingChains", renderProcessingChains()); + for (Extension extension : extensions) { + extension.produceExtraFields(this).forEach((field, json) -> { + if (root.has(field)) throw new IllegalArgumentException("Field '" + field + "' already defined"); + root.set(field, json); + }); - private static JsonNode renderProcessingChains(Container container) { - JsonNode ret = jsonMapper.createObjectNode(); - for (RequestHandler h : container.getRequestHandlerRegistry().allComponents()) { - if (h instanceof ProcessingHandler) { - ChainRegistry<Processor> registry = ((ProcessingHandler) h).getChainRegistry(); - return renderChains(registry); - } - } - return ret; } + return root; + } - // Note the generic param here! The key to make this work is '? extends Chain', but why? - static JsonNode renderChains(ComponentRegistry<? extends Chain<?>> chains) { - ObjectNode ret = jsonMapper.createObjectNode(); - for (Chain<?> chain : chains.allComponents()) { - ret.set(chain.getId().stringValue(), renderAbstractComponents(chain.components())); + private JsonNode renderProcessingChains() { + JsonNode ret = jsonMapper.createObjectNode(); + for (RequestHandler h : requestHandlers()) { + if (h instanceof ProcessingHandler) { + ChainRegistry<Processor> registry = ((ProcessingHandler) h).getChainRegistry(); + return renderChains(registry); } - return ret; } + return ret; + } - private static JsonNode renderCalls(Iterator<Call> components) { - ArrayNode ret = jsonMapper.createArrayNode(); - while (components.hasNext()) { - Call c = components.next(); - JsonNode jc = renderComponent(c.getDocumentProcessor(), c.getDocumentProcessor().getId()); - ret.add(jc); - } - return ret; + // Note the generic param here! The key to make this work is '? extends Chain', but why? + public static JsonNode renderChains(ComponentRegistry<? extends Chain<?>> chains) { + ObjectNode ret = jsonMapper.createObjectNode(); + for (Chain<?> chain : chains.allComponents()) { + ret.set(chain.getId().stringValue(), renderAbstractComponents(chain.components())); } - + return ret; } private class IgnoredContent implements ContentChannel { diff --git a/container-search-and-docproc/src/main/java/com/yahoo/container/handler/observability/package-info.java b/container-disc/src/main/java/com/yahoo/container/handler/observability/package-info.java index 7f9dc3d1cd9..1a669b176d0 100644 --- a/container-search-and-docproc/src/main/java/com/yahoo/container/handler/observability/package-info.java +++ b/container-disc/src/main/java/com/yahoo/container/handler/observability/package-info.java @@ -1,5 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ @ExportPackage package com.yahoo.container.handler.observability; -import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java index 33d357e8b6b..f04e2291ee8 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java @@ -76,6 +76,11 @@ public class AthenzIdentityProviderProvider implements Provider<AthenzIdentityPr } @Override + public X509Certificate getRoleCertificate(String domain, String role) { + throw new UnsupportedOperationException(message); + } + + @Override public PrivateKey getPrivateKey() { throw new UnsupportedOperationException(message); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java index 280e10f7022..d98ea0ffc25 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java @@ -2,6 +2,7 @@ package com.yahoo.container.jdisc; import ai.vespa.cloud.ApplicationId; +import ai.vespa.cloud.Cloud; import ai.vespa.cloud.Cluster; import ai.vespa.cloud.Environment; import ai.vespa.cloud.Node; @@ -34,6 +35,7 @@ public class SystemInfoProvider extends AbstractComponent implements Provider<Sy applicationIdConfig.application(), applicationIdConfig.instance()), new Zone(Environment.valueOf(csConfig.environment()), csConfig.region()), + new Cloud(csConfig.cloud()), new Cluster(ciConfig.nodeCount(), ciConfig.nodeIndices()), new Node(qrConfig.nodeIndex())); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java index c1c60612b37..af5133eceac 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java @@ -21,6 +21,7 @@ public interface AthenzIdentityProvider { String getAccessToken(String domain); String getAccessToken(String domain, List<String> roles); List<X509Certificate> getIdentityCertificate(); + X509Certificate getRoleCertificate(String domain, String role); PrivateKey getPrivateKey(); Path trustStorePath(); diff --git a/container-search-and-docproc/src/main/resources/configdefinitions/container.handler.observability.application-userdata.def b/container-disc/src/main/resources/configdefinitions/container.handler.observability.application-userdata.def index cde92df9ef4..cde92df9ef4 100644 --- a/container-search-and-docproc/src/main/resources/configdefinitions/container.handler.observability.application-userdata.def +++ b/container-disc/src/main/resources/configdefinitions/container.handler.observability.application-userdata.def diff --git a/container-search-and-docproc/src/test/java/com/yahoo/container/handler/observability/ApplicationStatusHandlerTest.java b/container-disc/src/test/java/com/yahoo/container/handler/observability/ApplicationStatusHandlerTest.java index 317f5fc1329..4c58a943199 100644 --- a/container-search-and-docproc/src/test/java/com/yahoo/container/handler/observability/ApplicationStatusHandlerTest.java +++ b/container-disc/src/test/java/com/yahoo/container/handler/observability/ApplicationStatusHandlerTest.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertTrue; /** * @author gjoranv - * @since 5.1.10 */ public class ApplicationStatusHandlerTest { @@ -114,7 +113,7 @@ public class ApplicationStatusHandlerTest { Chain<Processor> chain = new Chain<>("myChain", new VoidProcessor(new ComponentId("voidProcessor"))); chains.register(new ComponentId("myChain"), chain); - String json = ApplicationStatusHandler.StatusResponse.renderChains(chains).toString(); + String json = ApplicationStatusHandler.renderChains(chains).toString(); assertTrue(json.contains("myChain")); assertTrue(json.contains("voidProcessor")); } diff --git a/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java index 569d1439bbe..4264492ca2b 100644 --- a/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java +++ b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java @@ -47,8 +47,6 @@ public class GUIHandlerTest { public void testContentTypes() throws Exception{ assertResponse("/querybuilder/_includes/css/vespa.css", "/**","text/css; charset=UTF-8", 200); assertResponse("/querybuilder/js/agency.js", "/*!","application/javascript; charset=UTF-8", 200); - assertResponse("/querybuilder/img/reload.svg", "<?xml","image/svg+xml; charset=UTF-8", 200); - assertResponse("/querybuilder/img/Vespa-V2.png", null,"image/png; charset=UTF-8", 200); } @Test diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java index 8b827b3012d..351576cd38f 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java @@ -5,16 +5,14 @@ import com.yahoo.component.ComponentId; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.jdisc.service.CurrentContainer; -import java.util.logging.Level; import com.yahoo.messagebus.IntermediateSessionParams; import com.yahoo.messagebus.jdisc.MbusServer; import com.yahoo.messagebus.shared.SharedIntermediateSession; +import java.util.logging.Level; import java.util.logging.Logger; /** - * TODO: Javadoc - * * @author Tony Vaagenes */ public class MbusServerProvider implements Provider<MbusServer> { @@ -49,9 +47,12 @@ public class MbusServerProvider implements Provider<MbusServer> { @Override public void deconstruct() { log.log(Level.INFO, "Deconstructing mbus server: " + server); + long start = System.currentTimeMillis(); server.close(); server.release(); sessionRef.getReference().close(); + log.log(Level.INFO, String.format("Mbus server deconstruction completed in %.3f seconds", + (System.currentTimeMillis()-start)/1000D)); } } diff --git a/container-search-gui/src/main/resources/gui/_includes/css/vespa.css b/container-search-gui/src/main/resources/gui/_includes/css/vespa.css index a09364cefd4..19d5009e33f 100644 --- a/container-search-gui/src/main/resources/gui/_includes/css/vespa.css +++ b/container-search-gui/src/main/resources/gui/_includes/css/vespa.css @@ -8,9 +8,9 @@ --secondary-dark: #333; --muted: #777; - --fontprimary: HelveticaNeue,Helvetica,Arial,sans-serif; - --fontsecondary: HelveticaNeue-Thin,Helvetica,Arial,sans-serif; - }; + --fontprimary: "Hind Madurai",Helvetica,Arial,sans-serif; + --fontsecondary: "Hind Madurai",Helvetica,Arial,sans-serif; + } .bg-light-blue { background-image: linear-gradient(-1deg, rgba(63,157,216,0.08) 0%, rgba(163, 195, 215, 0.08) 97%); @@ -39,7 +39,7 @@ /*** Tooltips! ***/ .tip { visibility: visible; - border-bottom: 1px dotted [#FFFFFF]; + border-bottom: 1px dotted; position: relative; cursor: help; text-decoration: none; @@ -91,7 +91,7 @@ section h2.section-subheading, section h3.section-heading, section h3.section-subheading { text-transform: none; - font-family: HelveticaNeue-Thin,Helvetica,Arial,sans-serif; + font-family: "Hind Madurai",Helvetica,Arial,sans-serif; font-weight: normal; font-style: normal; } @@ -135,7 +135,7 @@ header .help-title { } .navbar-default { - background-color: #3F9DD8; + background-color: #005A8E; padding: 0; border-bottom: 2px solid rgba(255, 255, 255, 0.2); } @@ -146,14 +146,14 @@ header .help-title { } .navbar-default .navbar-brand { - background: transparent url("../../img/Vespa-V2.png") no-repeat; + background: transparent url("https://vespa.ai/assets/vespa-logo.png") no-repeat; background-size: contain; direction: ltr; text-indent: -9000px; height: 28px; width: 100px; display: inline-block; - font-family: HelveticaNeue,Helvetica,Arial,sans-serif; + font-family: "Hind Madurai",Helvetica,Arial,sans-serif; font-weight: bold; color: var(--primary); margin-top: 16px; @@ -166,7 +166,7 @@ header .help-title { letter-spacing: 1px; color: #FFFFFF; text-transform: none; - font-family: HelveticaNeue,Helvetica,Arial,sans-serif; + font-family: "Hind Madurai",Helvetica,Arial,sans-serif; } .navbar-default .navbar-nav > li > a:hover, @@ -189,7 +189,8 @@ section[id]:before { } header { - background-image: linear-gradient(0deg, #98C1DB 7%, #3F9DD8 100%); + /*background-image: linear-gradient(0deg, #98C1DB 7%, #3F9DD8 100%); */ + background-color: #005A8E; min-height: 1150px; } @@ -310,7 +311,7 @@ header .propvalue { margin-bottom: 3px; } -header .propvalue:-webkit-input-placeholder { /* Safari, Chrome(, Opera?) */ +header .propvalue::-webkit-input-placeholder { /* Safari, Chrome(, Opera?) */ color:gray; font-style:italic; } @@ -396,7 +397,6 @@ header .copyJSON{ height: 25px; border-width: 0px; border-radius: 5px; - padding: 0px padding-left: 1px; margin-top: 10px; margin-bottom: 20px; @@ -412,8 +412,7 @@ header .showJSON{ height: 25px; border-width: 0px; border-radius: 5px; - padding: 0px - padding-left: 1px; + padding: 0px; margin-top: 5px; margin-bottom: 10px; } @@ -426,13 +425,10 @@ header .pasteJSON{ height: 25px; border-width: 0px; border-radius: 5px; - padding: 0px; padding-left: 0px; padding-bottom: 2px; margin-top: 35px; margin-bottom: 10px; - - align-content: 30%; } @@ -444,7 +440,6 @@ header .copyURL{ height: 25px; border-width: 0px; border-radius: 5px; - padding: 0px padding-left: 1px; margin-top: 25px; margin-bottom: 10px; @@ -457,7 +452,7 @@ header .intro .intro-lead-in { font-size: 22px; line-height: 50px; margin-bottom: 25px; - font-family: HelveticaNeue-Thin,Helvetica,Arial,sans-serif; + font-family: "Hind Madurai",Helvetica,Arial,sans-serif; } header .intro .intro-heading { @@ -476,12 +471,12 @@ header .intro .btn-xl { color: #3F9DD8; font-weight: normal; text-transform: none; - font-family: HelveticaNeue,Helvetica,Arial,sans-serif; + font-family: "Hind Madurai",Helvetica,Arial,sans-serif; } header .intro-copy { - display:inline-block + display:inline-block; position: inherit; margin-top: 10px; padding-left: 50px; @@ -641,7 +636,7 @@ footer .credits { font-size: 12px; font-weight: normal; text-align: right; - font-family: HelveticaNeue-Thin,Helvetica,Arial,sans-serif; + font-family: "Hind Madurai",Helvetica,Arial,sans-serif; } footer .credits a { diff --git a/container-search-gui/src/main/resources/gui/_includes/index.html b/container-search-gui/src/main/resources/gui/_includes/index.html index bac486077a4..9a7c5e9ed33 100644 --- a/container-search-gui/src/main/resources/gui/_includes/index.html +++ b/container-search-gui/src/main/resources/gui/_includes/index.html @@ -1,15 +1,12 @@ <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <!DOCTYPE html> -<html> +<html lang="en"> <head> <meta charset="utf-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <title>Vespa. Big data. Real time.</title> + <title>Vespa Querybuilder</title> <meta name="viewport" content="width=device-width"> - <meta name="description" content=" Your big data processing and serving system - deep learning, - search applications, ad selection system, personalized - recommendation systems, and more..."> - + <meta name="description" content="Vespa Querybuilder - use to generate Vespa Query API requests"> <!-- Site icons - generated using http://realfavicongenerator.net/ --> <link rel="apple-touch-icon" sizes="180x180" href="/querybuilder/icons/apple-touch-icon.png"> @@ -27,161 +24,92 @@ <link href="https://fonts.googleapis.com/css?family=Lato:400,700,400italic,700italic" rel="stylesheet" type="text/css"> <link href='https://fonts.googleapis.com/css?family=Droid+Serif:400,700,400italic,700italic' rel='stylesheet' type='text/css'> <link href='https://fonts.googleapis.com/css?family=Roboto+Slab:400,100,300,700' rel='stylesheet' type='text/css'> + <link href='https://fonts.googleapis.com/css2?family=Hind+Madurai:wght@400;600;700' rel='stylesheet' type='text/css'> - <title>Vespa - Big Data. Real time.</title> - - <meta http-equiv="content-type" content="text/html; charset=utf-8"> - <meta name="viewport" content="width=device-width, initial-scale=1.0"> - <meta name="description" content=""> + <meta http-equiv="content-type" content="text/html; charset=utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <meta name="description" content=""> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script> - - </head> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script> +</head> <body id="page-top" class="index"> - -<style type="text/css"></style> - - <link href="/querybuilder/_includes/css/agency.css" rel="stylesheet" type="text/css"> <link href="/querybuilder/_includes/css/bootstrap.min.css" rel="stylesheet" type="text/css"> <link href="/querybuilder/_includes/css/vespa.css" rel="stylesheet" type="text/css"> <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- Brand and toggle get grouped for better mobile display --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <a class="navbar-brand" href="https://vespa.ai">Vespa. Big data. Real time.</a> - </div> + <div class="container"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="https://vespa.ai">Vespa.ai</a> + </div> - <!-- Collect the nav links, forms, and other content for toggling --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav navbar-right"> - <li class="hidden"><a href="#page-top"></a> - <li><a href="https://blog.vespa.ai/">Blog</a> - <li><a href="https://twitter.com/vespaengine">Twitter</a> - <li><a href="https://docs.vespa.ai">Docs</a> - <li><a href="https://github.com/vespa-engine">GitHub</a> - <li><a href="https://docs.vespa.ai/en/getting-started.html">Get Started Now</a> - </ul> - </div> + <!-- Collect the nav links, forms, and other content for toggling --> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right"> + <li><a href="https://docs.vespa.ai/en/query-api.html">Documentation</a></li> + <li><a href="https://github.com/vespa-engine/vespa/issues">Report issue</a></li> + </ul> </div> - </nav> + </div> +</nav> <!-- Header --> - <header> - <div class="layer"> - <div class="intro container"> - <div class="intro-lead-in">Vespa Search Engine</div> - <div class="intro-long">Select the method for sending requests and construct a query.</div> - <select class="methodselector" onchange="chooseMethod();" id="method"> - <option class="options" value="POST">POST</option> - <option class="options" value="GET">GET</option> - </select> - <input type="text" class="textbox" name="value" value="http://localhost:8080/search/" id="url" size="30"> - <button class="button" onclick="startSending();" id="send">Send</button> +<header> + <div class="layer"> + <div class="intro container"> + <div class="intro-lead-in">Vespa Querybuilder</div> + <div class="intro-long">Select the method for sending requests and construct a query:</div> + <select class="methodselector" onchange="chooseMethod();" id="method"> + <option class="options" value="POST">POST</option> + <option class="options" value="GET">GET</option> + </select> + <input type="text" class="textbox" name="value" value="http://localhost:8080/search/" id="url" size="30" /> + <button class="button" onclick="startSending();" id="send">Send</button> + + <br/><br/> - <br/> - <div id="request"> - </br> - <div class="intro-param">Construct a query by adding parameters or pasting a JSON.</div> - </div> - <br/> - <button class="addRow"onclick="addNewRow()" id="addRow">+</button> - <br/> - <button class="pasteJSON"onclick="pasteJSON();" id="pasteJSON"><img src="/querybuilder/img/paste.svg" height="16" width="16" style="margin-top:-2px; margin-right: 3px;"/> Paste JSON</button> - </br> - <button class="showJSON"onclick="showJSON();" id="showJSON">Show query JSON</button> - </br> - <textarea class="responsebox" id="jsonquery" cols=70 rows=15 style="display:none;"readonly></textarea> - </br> - <button class="copyJSON"onclick="copyToClipboard(jsonquery);" id="copyJSON"><img src="/querybuilder/img/copy.svg" height="17" width="17" /> Copy as JSON</button> - <button class="copyURL"onclick="copyURL();" id="copyURL"><img src="/querybuilder/img/copy.svg" height="17" width="17" /> Copy as URL</button> - <br/> + <div id="request"> + <div class="intro-param">Construct a query by adding parameters or pasting a JSON:</div> + </div> + <br/> + <button class="addRow" onclick="addNewRow()" id="addRow">+</button> + <br/> + <button class="pasteJSON" onclick="pasteJSON();" id="pasteJSON"> + <img src="/querybuilder/img/paste.svg" + height="16" width="16" style="margin-top:-2px; margin-right: 3px;" alt="paste"/> + Paste JSON + </button> + <button class="showJSON" onclick="showJSON();" id="showJSON">Show query JSON</button> + <textarea class="responsebox" id="jsonquery" cols=70 rows=15 style="display:none;"readonly></textarea> + <button class="copyJSON" onclick="copyToClipboard(jsonquery);" id="copyJSON"> + <img src="/querybuilder/img/copy.svg" height="17" width="17" alt="Copy as JSON"/> Copy as JSON + </button> + <button class="copyURL" onclick="copyURL();" id="copyURL"> + <img src="/querybuilder/img/copy.svg" height="17" width="17" alt="Copy as URL"/> Copy as URL + </button> + <br/> - <div class="response">Response</div> - <input class="responseinfo" id="reponsestatus" value="" readonly></input></br> - <textarea class="responsebox" id="response" cols=70 rows=25 readonly></textarea> - <div id="div" > - <div > + <div class="response">Response:</div> + <input class="responseinfo" id="reponsestatus" value="" readonly /> + <textarea class="responsebox" id="response" cols=70 rows=25 readonly></textarea> + <div id="div" > + <div> <div class="intro-copy" onclick="copyToClipboard(response);" style="cursor:pointer;"> - <img src="/querybuilder/img/copy.svg" height="30" width="30" /> - </div> - <div class="intro-refresh" onclick="refresh();" style="cursor:pointer;"> - <img src="/querybuilder/img/reload.svg" height="30" width="30" /> - </div> - </br> - </br> - <div id="helpbutton" class="intro-help" onclick="toggleHelp();" style="cursor:pointer;"> - <img src="/querybuilder/img/features-help.png" height="47" width="138" style="margin-left: -1px;"/> + <img src="/querybuilder/img/copy.svg" height="30" width="30" alt="Copy to clipboard"/> </div> - <div id="help" style="display: none;"> - <div class="intro-param" id="help"> - </br> - <div class="help-title">Features</div> - <span> ○ Autocompletion of YQL-syntax</span> </br> - <span> ○ Drop-down lists of all valid parameters</span> </br> - <span> ○ Sending both POST and GET-requests to <i>Vespa</i></span> </br> - <span> ○ Easy access to the <a href="https://docs.vespa.ai/en/reference/query-api-reference.html">documentation</a> of each parameter</span> </br> - <span> ○ Conversion of POST- to GET-query</span> </br> - <span> ○ Pasting already built JSON-query</span> </br> - <span> ○ View and copy the response of queries</span> </br> - <span> ○ View the finished JSON-query as you build </span> </br> - </div> - </br> - <div class="intro-param" id="help"> - <div class="help-title">Help</div> - <span>Please submit a <a href="https://github.com/vespa-engine/vespa/blob/master/container-search-gui/src/main/resources/gui/_includes/index.html">pull request</a> or create an <a href="https://github.com/vespa-engine/vespa/issues">issue</a> for fixes to the querybuilder.</span> </br> - </div> - </div> - </div> - </div> + </div> </div> </div> - </header> - <footer> - <div class="container"> - <div class="row"> - <div class="col-xs-2 quicklink-section"> - <div class="footer-title">Resources</div> - <ul class="quicklinks"> - <li><a href="https://docs.vespa.ai/en/vespa-quick-start.html">Getting Started</a> - <li><a href="https://docs.vespa.ai">Documentation</a> - <li><a href="https://github.com/vespa-engine/vespa">Open source</a> - </ul> - </div> - <div class="col-xs-2 quicklink-section"> - <div class="footer-title">Contact</div> - <ul class="quicklinks"> - <li><a href="https://twitter.com/vespaengine">Twitter</a> - <li><a href="mailto:info@vespa.ai">info@vespa.ai</a> - <li><a href="https://github.com/vespa-engine/vespa/issues">Issues</a> - </ul> - </div> - <div class="col-xs-2 quicklink-section"> - <div class="footer-title">Community</div> - <ul class="quicklinks"> - <li><a href="https://github.com/vespa-engine/vespa/blob/master/CONTRIBUTING.md">Contributing</a> - <li><a href="https://stackoverflow.com/questions/tagged/vespa">Stack Overflow</a> - <li><a href="https://gitter.im/vespa-engine/Lobby">Gitter</a> - </ul> - </div> - <div class="col-xs-6 quicklink-section"> - <div class="credits"> - <span>Copyright Yahoo</span> - Licensed under <a href="https://github.com/vespa-engine/vespa/blob/master/LICENSE">Apache License 2.0</a> - , <a href="https://github.com/y7kim/agency-jekyll-theme">Theme</a> by Rick K. - </div> - </div> - </div> - </div> - </footer> + </div> +</header> <!-- jQuery Version 1.11.0 --> <script src="/querybuilder/js/jquery-1.11.0.js"></script> @@ -229,7 +157,6 @@ childrenProps["ranking.properties"] = window.CONFIG.ranking_properties; } addNewRow(); - getSearchApiReference(); }, 250); }; var stringType = ["yql", "queryProfile", "searchChain", "model.defaultIndex", "model.encoding", "model.language", @@ -247,19 +174,6 @@ var yqlID = "v1"; - - function toggleHelp(){ - var div = document.getElementById("help"); - var buttonDiv = document.getElementById("helpbutton"); - if (div.style.display === "none") { - div.style.display = "block"; - buttonDiv.style.backgroundColor = '#4EA2D6'; - } else { - div.style.display = "none"; - buttonDiv.style.backgroundColor = 'transparent'; - } - } - function updateFields(){ var temp = number; while (temp > 0){ @@ -449,6 +363,7 @@ newButton.classList.add("removeRow"); var br = document.createElement("br"); br.id = "br"+temp + /* var img = document.createElement("img"); img.src = "/querybuilder/img/information.svg"; img.height = "15"; @@ -463,11 +378,12 @@ a.id = "inf"+temp a.appendChild(img); a.appendChild(span); + */ var div = document.createElement("div"); div.id = number; div.appendChild(newInput); div.appendChild(newDatalist); - div.appendChild(a); + //div.appendChild(a); div.appendChild(newInputVal); div.appendChild(newDatalist2); div.appendChild(newButton); @@ -490,6 +406,7 @@ generateJSON(); return temp; } + /* ToDo: remove completely function showInformation(no, key){ var a = document.getElementById("inf"+no); if(validKey(no, key)){ @@ -504,6 +421,7 @@ a.style = "visibility: hidden;" } } + */ function validKey(no, possibleKey){ if (contains(possible, possibleKey)){return true;} for (var key in childrenProps){ @@ -528,7 +446,7 @@ } function keySelected(no, value){ var key = document.getElementById("i"+no).value; - showInformation(no, key); + //showInformation(no, key); ToDo: remove completely findUsedProps(); var fullKey = getFullName(no, key); if(fullKey in childrenProps){ @@ -728,6 +646,7 @@ b.innerHTML = ' ↳ '; var margin = 20*(temp).length; b.style = "padding-left:"+margin+"px;"; + /* var img = document.createElement("img"); img.src = "/querybuilder/img/information.svg"; img.height = "15"; @@ -742,10 +661,11 @@ a.id = "inf"+temp a.appendChild(img); a.appendChild(span); + */ div.appendChild(b); div.appendChild(newInput); div.appendChild(newDatalist); - div.append(a); + //div.append(a); div.appendChild(newInputVal); div.appendChild(newDatalist2); div.appendChild(newButton); @@ -789,9 +709,7 @@ } return false; } - function refresh(){ - document.location.reload(true); - } + function generateJSON(){ json = JSON.parse("{}"); buildJSON(json, number, 0); @@ -940,20 +858,6 @@ copyURL.style.display = "none"; } } - function getSearchApiReference(){ - var div = document.getElementById("div"); - var object = document.createElement("object"); - object.data = "/querybuilder/_includes/search-api-reference.html"; - object.type ="text/html"; - object.style = "visibility: hidden; height: 1px; width: 200px;" - div.appendChild(object); - setTimeout(function(){ - searchApiReference = object.contentDocument; - setTimeout(function(){ - div.removeChild(object); - }, 20); - }, 150); - } function changeInformation(no,key){ if (key===""){return "Choose a parameter for information"} if (getFullName(no,key) in childrenProps){return "Add parameters under the parent ''" + getFullName(no,key) + "'"} diff --git a/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html b/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html deleted file mode 100644 index 086ff293b14..00000000000 --- a/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html +++ /dev/null @@ -1,1914 +0,0 @@ - -<!DOCTYPE html> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<html lang="en"> - -<head> - <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <!-- <meta name="viewport" content="width=device-width, initial-scale=1"> --> - <meta name="description" content="Serve big data with ease - dynamic responses at any scale in milliseconds with any traffic volume."> - <meta name="author" content="Vespa team"> - - <title>Search API Reference</title> - - <!-- Bootstrap --> - <link href="/css/bootstrap.min.css" rel="stylesheet"> - - <!-- Font Awesome --> - <link rel="stylesheet" href="/css/font-awesome.min.css"> - - <!-- Docs layout --> - <link rel="stylesheet" href="/css/docs.css"> - - <!-- Favicons --> - <link rel="icon" href="/icons/favicon.ico" type="image/x-icon" /> - <link rel="shortcut icon" href="/icons/favicon.ico" type="image/x-icon" /> - - <!-- Bootstrap and other JavaScript --> - <script type="text/javascript" async src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-MML-AM_CHTML"></script> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js"></script> - <script src="/js/bootstrap.min.js"></script> - - <!-- Global Site Tag (gtag.js) - Google Analytics --> - <script async src="https://www.googletagmanager.com/gtag/js?id=UA-107187180-3"></script> - <script> - window.dataLayer = window.dataLayer || []; - function gtag(){dataLayer.push(arguments)}; - gtag('js', new Date()); - gtag('config', 'UA-107187180-3'); -</script> - -</head> - -<body> - -<!-- Fixed navbar --> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<nav class="navbar navbar-default navbar-fixed-top"> - <div class="container-fluid"> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <a class="navbar-brand" href="http://vespa.ai"> - <img src="/img/vespa-logo.png" width="100" height="28"> - </a> - </div> - <div id="navbar" class="navbar-collapse collapse"> - <ul class="nav navbar-nav navbar-right"> - <li><a href="http://blog.vespa.ai/">Blog</a> - <li><a href="https://twitter.com/vespaengine">Twitter</a> - <li><a href="/documentation/">Docs</a> - <li><a href="https://github.com/vespa-engine" target="_blank">GitHub</a> - <li><a href="https://github.com/vespa-engine/vespa/issues" target="_blank">Issues</a> - </ul> - <div class="col-sm-offset-2 col-md-offset-2"> - <div class="row"> - <form class="search" action="/search.html" method="get" id="search-form"> - <div class="col-xs-4"> - <input type="text" class="form-control" placeholder="Search Documentation" id="search" name="q"> - </div> - <button type="submit" id="submit" class="btn btn-search"><i class="fa fa-search" aria-hidden="true"></i></button> - </form> - </div> - </div> - </div> - </div> -</nav> - - -<div id="main-content" class="container-fluid"> - <div class="row"> - - <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> - - <div class="col-sm-3 col-md-2 sidebar"> - <ul class="nav nav-sidebar"> - <li class="collapse-all" onclick="expand_all();">[+] expand all</li> - </ul> - - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">WELCOME</li> - - - - <li class="collapseable"><a href="/">Welcome to Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/overview.html">Vespa Overview</a></li> - - - - - <li class="collapseable"><a href="/documentation/features.html">Features</a></li> - - - - - <li class="collapseable"><a href="/documentation/elastic-search-comparison.html">Comparison to Elasticsearch</a></li> - - - - - <li class="collapseable"><a href="/documentation/contributing.html">Contributing to Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/introduction-to-documentation.html">Documentation Conventions</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">GETTING STARTED</li> - - - - <li class="collapseable"><a href="/documentation/vespa-quick-start.html">Quick Start</a></li> - - - - - <li class="collapseable"><a href="/documentation/tutorials/blog-search.html">Blog Search Tutorial</a></li> - - - - - <li class="collapseable"><a href="/documentation/build-install-vespa.html">Build Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/api.html">Vespa API and Interfaces</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">DOCUMENTS AND SEARCH DEFINITIONS</li> - - - - <li class="collapseable"><a href="/documentation/documents.html">Documents</a></li> - - - - - <li class="collapseable"><a href="/documentation/search-definitions.html">Search Definitions</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">WRITING TO VESPA</li> - - - - <li class="collapseable"><a href="/documentation/writing-to-vespa.html">Writing to Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/vespa-feed-client.html">Vespa Feeding Client API</a></li> - - - - - <li class="collapseable"><a href="/documentation/feed-using-hadoop-pig-oozie.html">Feed using Hadoop, Pig, Oozie</a></li> - - - - - <li class="collapseable"><a href="/documentation/document-api-guide.html">Document API</a></li> - - - - - <li class="collapseable"><a href="/documentation/document-processing-overview.html">Document Processing</a></li> - - - - - <li class="collapseable"><a href="/documentation/annotations.html">Annotations API</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">QUERYING VESPA</li> - - - - <li class="collapseable"><a href="/documentation/querying-vespa.html">Querying Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/search-api.html">Search API</a></li> - - - - - <li class="collapseable"><a href="/documentation/query-language.html">Vespa Query Language</a></li> - - - - - <li class="collapseable"><a href="/documentation/grouping.html">Grouping Information in Results</a></li> - - - - - <li class="collapseable"><a href="/documentation/federation.html">Federation</a></li> - - - - - <li class="collapseable"><a href="/documentation/query-profiles.html">Query Profiles</a></li> - - - - - <li class="collapseable"><a href="/documentation/geo-search.html">Geo Search</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">RANKING AND ML MODELS</li> - - - - <li class="collapseable"><a href="/documentation/ranking.html">Ranking Introduction</a></li> - - - - - <li class="collapseable"><a href="/documentation/nativerank.html">Ranking With nativeRank</a></li> - - - - - <li class="collapseable"><a href="/documentation/tensor-intro.html">Tensor Introduction</a></li> - - - - - <li class="collapseable"><a href="/documentation/tensor-user-guide.html">Tensor User Guide</a></li> - - - - - <li class="collapseable"><a href="/documentation/tensorflow.html">Ranking With TensorFlow Models</a></li> - - - - - <li class="collapseable"><a href="/documentation/onnx.html">Ranking With ONNX Models</a></li> - - - - - <li class="collapseable"><a href="/documentation/xgboost.html">Ranking With XGBoost Models</a></li> - - - - - <li class="collapseable"><a href="/documentation/stateless-model-evaluation.html">Stateless model evaluation</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">LINGUISTICS AND TEXT PROCESSING</li> - - - - <li class="collapseable"><a href="/documentation/linguistics.html">Linguistics in Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/stemming.html">Stemming</a></li> - - - - - <li class="collapseable"><a href="/documentation/text-processing.html">Special Tokens</a></li> - - - - - <li class="collapseable"><a href="/documentation/query-rewriting.html">Query Rewriting</a></li> - - - - - <li class="collapseable"><a href="/documentation/query-phrasing.html">Query Phrasing With PhrasingSearcher</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">DEVELOPING APPLICATIONS AND PLUGINS</li> - - - - <li class="collapseable"><a href="/documentation/cloudconfig/application-packages.html">Application Packages</a></li> - - - - - <li class="collapseable"><a href="/documentation/container-intro.html">The Vespa Container</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/">Java Data Intensive Serving Container - JDisc</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/container-components.html">JDisc Container Components</a></li> - - - - - <li class="collapseable"><a href="/documentation/component-types.html">Container Component Types</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/developing-applications.html">Application Development Basics</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/processing.html">Request-Response Processing</a></li> - - - - - <li class="collapseable"><a href="/documentation/searcher-development.html">Searcher Development</a></li> - - - - - <li class="collapseable"><a href="/documentation/docproc-development.html">Document Processor Development</a></li> - - - - - <li class="collapseable"><a href="/documentation/developing-web-services.html">Developing Web Service Applications</a></li> - - - - - <li class="collapseable"><a href="/documentation/handler-tutorial.html">HTTP API Use Case Tutorial</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/injecting-components.html">Component Injection</a></li> - - - - - <li class="collapseable"><a href="/documentation/chained-components.html">Chained Components</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/developing-osgi-bundles.html">Building OSGi Bundles</a></li> - - - - - <li class="collapseable"><a href="/documentation/bundle-plugin.html">OSGi Bundle Maven Plugin</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/component-versioning.html">Versioning in the Container</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/metrics.html">Container Metrics</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">CONFIGURATION</li> - - - - <li class="collapseable"><a href="/documentation/cloudconfig/config-introduction.html">The Cloud Configuration System</a></li> - - - - - <li class="collapseable"><a href="/documentation/cloudconfig/configapi-dev.html">Cloud Config API</a></li> - - - - - <li class="collapseable"><a href="/documentation/cloudconfig/cloudconfig-model-plugins.html">Developing Cloud Config Model Plugins</a></li> - - - - - <li class="collapseable"><a href="/documentation/cloudconfig/deploy-rest-api-v2.html">Deploy API</a></li> - - - - - <li class="collapseable"><a href="/documentation/cloudconfig/config-rest-api-v2.html">Config API</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">RESULT FORMATS</li> - - - - <li class="collapseable"><a href="/documentation/document-summaries.html">Document Summaries</a></li> - - - - - <li class="collapseable"><a href="/documentation/result-rendering.html">Search Result Renderers</a></li> - - - - - <li class="collapseable"><a href="/documentation/page-templates.html">Page Templates</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">PREDICATE SEARCH</li> - - - - <li class="collapseable"><a href="/documentation/predicate-fields.html">Predicate Fields</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">USER AND GROUP ORIENTED SEARCH</li> - - - - <li class="collapseable"><a href="/documentation/streaming-search.html">Streaming Search</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">THE CONTENT LAYER</li> - - - - <li class="collapseable"><a href="/documentation/elastic-vespa.html">Elastic Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/proton.html">Proton</a></li> - - - - - <li class="collapseable"><a href="/documentation/content/consistency.html">Vespa Consistency Model</a></li> - - - - - <li class="collapseable"><a href="/documentation/content/idealstate.html">Distribution Algorithm</a></li> - - - - - <li class="collapseable"><a href="/documentation/content/data-placement.html">Document Distribution</a></li> - - - - - <li class="collapseable"><a href="/documentation/content/buckets.html">Buckets</a></li> - - - - - <li class="collapseable"><a href="/documentation/content/admin-states.html">Cluster and Node States</a></li> - - - - - <li class="collapseable"><a href="/documentation/content/api-state-rest-api.html">State API</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">PERFORMANCE AND TUNING</li> - - - - <li class="collapseable"><a href="/documentation/performance/sizing-search.html">Vespa Search Sizing Guide</a></li> - - - - - <li class="collapseable"><a href="/documentation/attributes.html">Document Attributes</a></li> - - - - - <li class="collapseable"><a href="/documentation/performance/attribute-memory-usage.html">Attribute Memory Usage</a></li> - - - - - <li class="collapseable"><a href="/documentation/performance/vespa-benchmarking.html">Vespa Benchmarking</a></li> - - - - - <li class="collapseable"><a href="/documentation/performance/profiling-search-container.html">Profiling the Search Container</a></li> - - - - - <li class="collapseable"><a href="/documentation/performance/container-tuning.html">Container Tuning</a></li> - - - - - <li class="collapseable"><a href="/documentation/performance/rate-limiting-searcher.html">Rate-Limiting Search Requests</a></li> - - - - - <li class="collapseable"><a href="/documentation/performance/fbench.html">Vespa HTTP Benchmark Tool - vespa-fbench</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">OPERATIONS AND PROCEDURES</li> - - - - <li class="collapseable"><a href="/documentation/operations/admin-procedures.html">Administrative Procedures</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/vespa-cmdline-tools.html">Vespa Command-line Tools</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/metrics-health-format.html">Gathering Metrics from Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/logs.html">Log Files</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/files-processes-and-ports.html">Files, Processes and Ports</a></li> - - - - - <li class="collapseable"><a href="/documentation/inspecting-java-services.html">Inspecting Vespa Java Services</a></li> - - - - - <li class="collapseable"><a href="/documentation/docker-containers-in-production.html">Docker Containers in Production</a></li> - - - - - <li class="collapseable"><a href="/documentation/securing-your-vespa-installation.html">Securing a Vespa installation</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">MIGRATING FROM ELASTICSEARCH TO VESPA</li> - - - - <li class="collapseable"><a href="/documentation/migrating-from-elastic-search-to-vespa.html">Migrating from Elasticsearch to Vespa</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">CONFIGURATION REFERENCE</li> - - - - <li class="collapseable"><a href="/documentation/reference/application-packages-reference.html">Application Package Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/search-definitions-reference.html">Search Definition Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/services.html">Services Configuration (services.xml)</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/advanced-indexing-language.html">Indexing Language</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/config-files.html">Custom Configuration File Reference</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">RANKING REFERENCE</li> - - - - <li class="collapseable"><a href="/documentation/reference/ranking-expressions.html">Ranking Expressions</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/tensor.html">Tensor Evaluation Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/nativerank.html">nativeRank Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/rank-features.html">Rank Feature Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/string-segment-match.html">String Segment Match</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/rank-feature-configuration.html">Rank Feature Configuration</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/rank-types.html">Rank Types</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">QUERIES AND RESULTS REFERENCE</li> - - - - <li class="active collapseable"><a href="/documentation/reference/search-api-reference.html">Search API Reference <span class="sr-only">(current)</span></a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/simple-query-language-reference.html">Simple Query Language Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/select-reference.html">Select Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/grouping-syntax.html">Grouping Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/sorting.html">Sorting Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/query-profile-reference.html">Query Profile Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/semantic-rules.html">Semantic Rule Language Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/advanced-search-operators.html">Advanced Search Operators</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/equiv.html">The EQUIV Operator</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/wand-operator.html">The WAND Operator</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/weighted-set-term.html">The WeightedSetItem Operator</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/dot-product-search-operator.html">The Dot Product Search Operator</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/default-result-format.html">Default JSON Result Format</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/page-templates-syntax.html">Page Templates Syntax</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">DOCUMENT FORMAT AND LANGUAGES REFERENCE</li> - - - - <li class="collapseable"><a href="/documentation/reference/document-json-format.html">Document JSON Format</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/document-field-path.html">Document Field Path Syntax</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/document-select-language.html">Document Selector Language</a></li> - - - - - </ul> - - - </div> - - <script type="application/javascript"> - - $(document).ready (function() { - $(".collapseable").each(function() { - $(this).addClass("collapsed"); - }); - $(".active").each(function() { - $(this).removeClass("collapsed"); - $(this).siblings().each(function() { - if (!$(this).hasClass("collapse-parent")) { - $(this).removeClass("collapsed"); - } - }); - }); - }); - - function on_collapse(e) { - $(e).siblings().each(function() { - $(this).toggleClass("collapsed"); - }); - } - - function expand_all() { - $(".collapseable").each(function() { - $(this).removeClass("collapsed"); - }); - } - -</script> - - - <div class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main"> - - <h1> - Search API Reference - <div class="pull-right"><a href="https://github.com/vespa-engine/documentation/blob/master/documentation/reference/search-api-reference.html" class="btn btn-link"><i class="fa fa-github"></i> edit page</a></div> - </h1> - - <p> - All the search request parameters listed below can be set in query - profiles. The first four blocks of properties are also modeled as - query profile types. These types can be referred from query profiles - (and inheriting types) to provide type checking on the parameters. - </p><p> - These parameters often have both a full name - which includes the - path from the root query profile - and one or more abbreviated - names. Both names can be used in search requests, while only full - names can be used in query profiles. The full names are case - sensitive, while the abbreviated names are case insensitive. - </p><p> - The parameters modeled as query profiles are also available through - get methods as Java objects from the Query to Searcher components. - </p> - - - - <h2>Index</h2> - - <dt>Query</dt> - <dd> - <ul> - <li><a href="#yql">yql</a></li> - <li><a href="#Select">select</a></li> - <ul> - <li><a href="#where">where</a></li> - <li><a href="#grouping">grouping</a></li> - </ul> - </ul> - </dd> - </dl> - - <dl> - <dt>Native Execution Parameters</dt> - <dd> - <ul> - <li><a href="#hits">hits</a> [<em>count</em>]</li> - <li><a href="#offset">offset </a>[<em>start</em>]</li> - <li><a href="#queryProfile">queryProfile</a></li> - <li><a href="#nocache">nocache</a></li> - <li><a href="#groupingSessionCache">groupingSessionCache</a></li> - <li><a href="#searchChain">searchChain</a></li> - <li><a href="#timeout">timeout</a></li> - <li><a href="#tracelevel">tracelevel</a></li> - <li><a href="#trace.timestamps">trace.timestamps</a></li> - </ul> - </dd> - - <dt>Query Model Parameters</dt> - <dd> - <ul> - <li><a href="#model.defaultIndex">model.defaultIndex</a> [<em>default-index</em>]</li> - <li><a href="#model.encoding">model.encoding</a> [<em>encoding</em>]</li> - <li><a href="#model.filter">model.filter</a> [<em>filter</em>]</li> - <li><a href="#model.language">model.language</a> [<em>lang, language</em>]</li> - <li><a href="#model.queryString">model.queryString</a> [<em>query</em>]</li> - <li><a href="#model.restrict">model.restrict</a> [<em>restrict</em>]</li> - <li><a href="#model.searchPath">model.searchPath</a> [<em>path</em>]</li> - <li><a href="#model.sources">model.sources</a> [<em>search, sources</em>]</li> - <li><a href="#model.type">model.type</a> [<em>type</em>]</li> - </ul> - </dd> - - <dt>Ranking</dt> - <dd> - <ul> - <li><a href="#ranking.location">ranking.location</a> [<em>location</em>]</li> - <li><a href="#ranking.features">ranking.features</a> [<em>rankfeature</em>]</li> - <li><a href="#ranking.listFeatures">ranking.listFeatures</a> [<em>rankfeatures</em>]</li> - <li><a href="#ranking.profile">ranking.profile</a> [<em>ranking</em>]</li> - <li><a href="#ranking.properties">ranking.properties</a> [<em>rankproperty</em>]</li> - <li><a href="#ranking.sorting">ranking.sorting</a> [<em>sorting</em>]</li> - <li><a href="#ranking.freshness">ranking.freshness</a></li> - <li><a href="#ranking.queryCache">ranking.queryCache</a></li> - <li><a href="#ranking.matchPhase">ranking.matchPhase</a></li> - </ul> - </dd> - - <dt>Presentation</dt> - <dd> - <ul> - <li><a href="#presentation.bolding">presentation.bolding</a> [<em>bolding</em>]</li> - <li><a href="#presentation.format">presentation.format</a> [<em>format</em>]</li> - <li><a href="#presentation.template">presentation.template</a></li> - <li><a href="#presentation.summary">presentation.summary</a> [<em>summary</em>]</li> - <li><a href="#presentation.timing">presentation.timing</a></li> - </ul> - </dd> - - <dt>Grouping</dt> - <dd> - <ul> - <li><a href="#select">select</a></li> - <li><a href="#collapse.summary">collapse.summary</a></li> - <li><a href="#collapsefield">collapsefield</a></li> - <li><a href="#collapsesize">collapsesize</a></li> - </ul> - </dd> - - <dt>Geographical Searches</dt> - <dd> - <ul> - <li><a href="#pos.ll">pos.ll</a></li> - <li><a href="#pos.radius">pos.radius</a>,</li> - <li><a href="#pos.attribute">pos.attribute</a></li> - <li><a href="#pos.bb">pos.bb</a></li> - </ul> - </dd> - - <dt>Streaming Search</dt> - <dd> - <ul> - <li><a href="#streaming.userid">streaming.userid</a></li> - <li><a href="#streaming.groupname">streaming.groupname</a></li> - <li><a href="#streaming.selection">streaming.selection</a></li> - <li><a href="#streaming.priority">streaming.priority</a></li> - <li><a href="#streaming.maxbucketspervisitor">streaming.maxbucketspervisitor</a></li> - </ul> - </dd> - - <dt>Semantic Rules</dt> - <dd> - <ul> - <li><a href="#rules.off">rules.off</a></li> - <li><a href="#rules.rulebase">rules.rulebase</a></li> - <li><a href="#tracelevel.rules">tracelevel.rules</a></li> - </ul> - </dd> - - <dt>Other</dt> - <dd> - <ul> - <li><a href="#recall">recall</a></li> - <li><a href="#user">user</a></li> - <li><a href="#nocachewrite">nocachewrite</a></li> - <li><a href="#hitcountestimate">hitcountestimate</a></li> - <li><a href="#metrics.ignore">metrics.ignore</a></li> - </ul> - </dd> - </dl> - - - <h2 id="query">Query</h2> - <h3 id="yql">yql</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>String</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - The YQL query will be parsed and executed in the backend. - Only simple YQL programs are supported, refer to - <a href="../query-language.html">YQL</a> for details. - </p> - - <h3 id="Select">select</h3> - <p>Select query is equivalent with YQL, written in JSON. Contains subparameters <code>where</code> and <code>grouping</code>.</p> - - <h4 id="where"> where</h4> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>JSON</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - - <h4 id="grouping"> grouping</h4> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>JSON</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - The where and grouping query will be parsed and executed in the backend. - Refer to - <a href="../reference/select-reference.html">Select Reference</a> for details. - </p> - - - - - <h2 id="native-execution-parameters">Native Execution Parameters</h2> - <p> - These parameters are defined in the <code>native</code> query profile type. - </p> - - - <h3 id="hits">hits</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>count</td></tr> - <tr><td>Values</td> - <td> - A positive integer, or 0. The sum of <a href="#offset">offset</a> and - <a href="#hits">hits</a> should be lower than the configured maxoffset - value, and will be adjusted to fit. See also comment - at <code>offset</code>. - </td> - </tr> - <tr><td>Default</td><td>10</td></tr> - </table> - <p> - The maximum number of hits to return from the result set. - Must be lower than <code>maxHits</code>, which is either set in a - <a href="#queryProfile">query profile</a>, or default 400. - <!-- ToDo: link to def file or code where this is definied --> - </p> - - - <h3 id="offset">offset</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>start</td></tr> - <tr><td>Values</td> - <td> - A positive integer, including 0. - </td> - </tr> - <tr><td>Default</td><td>0</td></tr> - </table> - <p> - The index of the first hit to return from the result set. - Must be lower than <code>maxOffset</code>, which is either set in a - <a href="#queryProfile">query profile</a>, or default 1000. - <!-- ToDo: link to def file or code where this is definied --> - </p> - - - <h3 id="queryProfile">queryProfile</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td><em>None</em></td></tr> - <tr><td>Values</td> - <td> - A query profile id - name:version, where version can be omitted - or partially specified, e.g "myprofile:2.1" - </td> - </tr> - <tr><td>Default</td><td><code>default</code></td></tr> - </table> - <p> - A <a href="../query-profiles.html">query profile</a> has default properties for a query. - The default query profile is named <em>default</em> - example: - <pre> -<query-profile id="default"> - <field name="maxHits">10</field> - <field name="maxOffset">1000</field> -</query-profile> -</pre> - </p> - - - <h3 id="nocache">nocache</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td> - True or false - </td> - </tr> - <tr><td>Default</td><td>false</td></tr> - </table> - <p> - Set to true to avoid the result being fetched from cache, and avoid - writing the result to cache after fetching it. - </p> - - - <h3 id="groupingSessionCache">groupingSessionCache</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td> - True or false - </td> - </tr> - <tr><td>Default</td><td>false</td></tr> - </table> - <p> - Set to true to store intermediate grouping results in the search back ends when - using multi level grouping expressions in order to speed up grouping at a - potential loss of accuracy. See the <a - href="grouping-syntax.html#sessionCache">grouping reference</a> for more - details. - </p> - - - <h3 id="searchChain">searchChain</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td> - A search chain id - name:version, where version can be - omitted or partially specified, e.g "mychain:2.1.3". - </td> - </tr> - <tr><td>Default</td><td><code>default</code></td></tr> - </table> - <p> - The search chain initially invoked when processing this query. This - search chain may invoke other chains. - </p> - - - <h3 id="timeout">timeout</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td> - Positive floating point number with an optional unit. Default unit - is seconds (s), valid unit strings are e.g. <em>ms</em> and <em>s</em>. To set - a timeout of one minute, the argument could be set to <em>60 s</em>. - Space between the number and the unit is optional. - </td> - </tr> - <tr><td>Default</td><td>Undefined, but guaranteed to be at least 5000 milliseconds. This default can be overridden by configuring timeout in a <a href="../query-profiles.html">query profile.</a></td></tr> - </table> - <p>The query timeout.</p> - - - <h3 id="tracelevel">tracelevel</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td> - Any positive number - </td> - </tr> - <tr><td>Default</td><td><em>No tracing</em></td></tr> - </table> - <p> - Set to a positive number to collect trace information for debugging - when running a query. Higher numbers give - progressively more detail on query transformations and searcher - execution. - </p> - - <h3 id="trace.timestamps">trace.timestamps</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td> - true or false - </td> - </tr> - <tr><td>Default</td><td><em>No timestamps in trace</em></td></tr> - </table> - <p> - Enable it to get timing information already at <a href="#tracelevel">tracelevel=1</a> which is useful for debugging latency spent at different components in the search chain without rendering a lot of string data which is associated with higher trace levels. - </p> - - - - <h2 id="query-model">Query Model Parameters</h2> - - <h3 id="model.defaultIndex">model.defaultIndex [<em>default-index</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>default-index</td></tr> - <tr><td>Values</td><td>An index name</td></tr> - <tr><td>Default</td><td><code>default</code></td></tr> - </table> - <p> - The field which is searched for query terms which doesn't explicitly specify an index. - </p> - - - <h3 id="model.encoding">model.encoding [<em>encoding</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>encoding</td></tr> - <tr><td>Values</td><td>Encoding names or aliases defined in the <a href="http://www.iana.org/assignments/character-sets">IANA character sets</a></td></tr> - <tr><td>Default</td><td>utf-8</td></tr> - </table> - <p> Sets the encoding to use when returning a result. The encodings <em>big5</em>, - <em>euc-jp</em>, <em>euc-kr</em>, <em>gb2312</em>, <em>iso-2022-jp</em> and <em>shift-jis</em> - also influences how tokenization is done in the absence of an explicit language setting. - </p><p> - The query is always encoded as UTF-8, independently of how the result will be encoded. - </p> - - - <h3 id="model.filter">model.filter [<em>filter</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>filter</td></tr> - <tr><td>Values</td><td>Any allowed collection of filter terms</td></tr> - <tr><td>Default</td><td><em>Not set</em></td></tr> - </table> - <p> - Sets a filter to be combined with the query. Typical use of a filter - is to add machine generated or preferences based filter terms to a raw - user query. The filter is parsed the same way as a query of type any, - the full syntax is available. The positive terms (preceded by +) and - phrases act as AND filters, the negative terms (preceded by -) act as - NOT filters, while the unprefixed terms will be used to RANK the - results. Unless the query has no positive terms, the filter will only - restrict and influence ranking of the result set, never cause more - matches than the query. - </p> - - - <h3 id="model.language">model.language [<em>lang, language</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>language, lang</td></tr> - <tr><td>Values</td><td>Ref. RFC 3066</td></tr> - <tr><td>Default</td><td></em>Unspecified</em></td></tr> - </table> - <p> - Informs Vespa about the natural language of the query. Please see - <a href="../linguistics.html">linguistics</a> for details. - This attribute should always be set when it is known. If this - parameter is not set, it will be guessed from the query and encoding, and - default to english if it cannot be guessed. - </p> - - - <h3 id="model.queryString">model.queryString [<em>query</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>query</td></tr> - <tr><td>Values</td><td>Any HTTP encoded legal Vespa query language string</td></tr> - <tr><td>Default</td><td><em>Not set</em></td></tr> - </table> - <p> - The <a href="simple-query-language-reference.html">Simple Vespa Query Language</a> query string - specifying which documents to match in this query. - </p> - - - <h3 id="model.restrict">model.restrict [<em>restrict</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>restrict</td></tr> - <tr><td>Values</td><td>A comma delimited list of document type names.</td></tr> - <tr><td>Default</td><td><em>Search unrestricted</em></td></tr> - </table> - <p> - The document types to restrict the search to when different document - types share the same search cluster. - </p> - - - <h3 id="model.searchPath">model.searchPath [<em>path</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>searchpath</td></tr> - <tr><td>Values</td><td><ul> - <li>searchpath::ELEMENT [';' ELEMENT]*</li> - <li>ELEMENT::PART ['/' ROW]</li> - <li>PART::EXP [',' EXP]*</li> - <li>EXP::NUM | RANGE</li> - <li>ROW::NUM</li> - <li>RANGE::'['NUM ',' NUM ' >'</li> - </ul></td></tr> - <tr><td>Default</td><td><em>Whole cluster</em></td></tr> - </table> - <p> - Specification of which path to send the query to. - Used to select which set of search nodes in the cluster should be used. - Only meant for debugging/monitoring. - </p><p> - Examples: - Note that in an indexed content cluster with flat distribution we have 1 implicit row - and each search node represents a part. - <ul> - <li>'7/3' = part 7, row 3.</li> - <li>'7/' = part 7, any row.</li> - <li>'7,1,9/0' = parts 1,7 and 9, row 0.</li> - <li>'1,[3,9>/0' = parts 1,3,4,5,6,7,8, row 0.</li> - </ul> - </p><p> - In a cluster with a multi-level dispatch setup we must specify a search path element for each level. - Lets say we have a setup with 2 mid-level dispatch groups, each containing 3 search nodes (and 3 dispatchers): - <ul> - <li>'0/;2/' = dispatch group (part) 0, any of the dispatchers (row); search node (part) 2, any row (of 1 present).</li> - <li>'0/1;2/0' = dispatch group (part) 0, dispatcher (row) 1; search node (part) 2, row 0 (of 1 present).</li> - </ul> - </p> - - - <h3 id="model.sources">model.sources [<em>search, sources</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>search, sources</td></tr> - <tr><td>Values</td><td>A comma separated list of search cluster names or other source names</td></tr> - <tr><td>Default</td><td><em>Search unrestricted</em></td></tr> - </table> - <p> - The names of the sources to search, e.g one or more search clusters and/or federated sources. - </p> - - - <h3 id="model.type">model.type [<em>type</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>type</td></tr> - <tr><td>Values</td><td>web, all, any, phrase, yql, adv (deprecated) - - refer to <a href="simple-query-language-reference.html">simple query language reference</a></td></tr> - <tr><td>Default</td><td>all</td></tr> - </table> - <p> - Selects the query language syntax of the <a href="#model.queryString">query</a> parameter. - </p> - - - - <h2 id="ranking">Ranking</h2> - - <h3 id="ranking.location">ranking.location [<em>location</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>location</td></tr> - <tr><td>Values</td><td>See <a href="../geo-search.html">Geo search</a></td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - Point (one or two dimensional) location to use as base for location ranking. - For geographical locations, it is recommended to add the location using <a href="#pos.ll">pos.ll</a> - <!-- ToDo: Why? --> - </p> - - - <h3 id="ranking.features">ranking.features.<em>featurename</em> [<em>rankfeature.</em>featurename]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>rankfeature.featurename</td></tr> - <tr><td>Values</td><td>Any string</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - Set a rank feature to a value. This works for any key name <code>query(anyname)</code> (query features), - and also as a way to override all existing (match and document) features. - Example: <em>query=foo&ranking.features.query(userage)=42&ranking.features.fieldMatch(title)=0.65</em> - </p> - - - <h3 id="ranking.listFeatures">ranking.listFeatures [<em>rankfeatures</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>rankfeatures</td></tr> - <tr><td>Values</td><td>boolean</td></tr> - <tr><td>Default</td><td>false</td></tr> - </table> - <p> - Set to true to request <em>all</em> rank features to be calculated and returned. - The rank features will be returned in the summary field rankfeatures. - This option is typically used for MLR training, should not to be used for production. - </p> - - - <h3 id="ranking.profile">ranking.profile [<em>ranking</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>ranking</td></tr> - <tr><td>Values</td><td>Any rank profile name</td></tr> - <tr><td>Default</td><td><code>default</code></td></tr> - </table> - <p> - Sets the name of the rank profile to use for assigning relevancy scores. - The default rank profile will be used for back-ends which does not have the given rank profile. - </p> - - - <h3 id="ranking.properties">ranking.properties.<em>propertyname</em> [<em>rankproperty.propertyname</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>rankproperty.propertyname</td></tr> - <tr><td>Values</td><td>Any string</td></tr> - <tr><td>Default</td><td><em>None</em></td></tr> - </table> - <p> - Set a rank property that is passed to, and used by a feature executor for this query. - Example: <em>query=foo&ranking.properties.dotProduct.X={a:1,b:2}</em> - </p> - - - <h3 id="ranking.sorting">ranking.sorting [<em>sorting</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>sorting</td></tr> - <tr><td>Values</td><td>A valid <a href="sorting.html">sort specification</a></td></tr> - <tr><td>Default</td><td>None - order by relevance</td></tr> - </table> - <p> - A specification of how to sort the result. - Fields you want to sort on must be stored as document attributes in the index structure - by adding <a href="search-definitions-reference.html#attribute">attribute</a> to the indexing statement. - </p> - - - <h3 id="ranking.freshness">ranking.freshness</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td><code>[integer]</code>, an absolute time in seconds since epoch, or <code>now-[number]</code>, to use a time [integer] seconds into the past, or <code>now</code> to use the current time</td></tr> - <tr><td>Default</td><td>None - use the current time on each node.</td></tr> - </table> - <p> - Sets the time which will be used as <em>now</em> during execution. - </p> - - - <h3 id="ranking.queryCache">ranking.queryCache</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>boolean</td></tr> - <tr><td>Default</td><td>false</td></tr> - </table> - <p> - Turns query cache on or off. Search is a two-phase process. If the - query cache is on, the query is stored on the search nodes between the - first and second phase, saving network bandwidth and also query setup - time, at the expense of using more memory. - </p> - - - <h3 id="ranking.matchPhase">ranking.matchPhase</h3> - <p>Settings which control Vespa's behavior during the match phase. - If these are set in the query they will override any match-phase setting - in the rank profile.</p> - <dt></dt> - <dd> - <ul> - <li><a href="#ranking.matchPhase.maxHits">ranking.matchPhase.maxHits</a> the max number of hits that should be generated during the match phase</li> - <li><a href="#ranking.matchPhase.attribute">ranking.matchPhase.attribute</a> the attribute to limit matches by if more than maxHits hits will be generated</li> - <li><a href="#ranking.matchPhase.ascending">ranking.matchPhase.ascending</a> whether to keep the documents having the highest (default) or lowest values of the attribute</li> - <li><a href="#ranking.matchPhase.diversity.attribute">ranking.matchPhase.diversity.attribute</a> the attribute to use to guarantee diversity.</li> - <li><a href="#ranking.matchPhase.diversity.minGroups">ranking.matchPhase.diversity.minGroups</a> the minimum number of groups grouped by the diversity attribute.</li> - </ul> - </dd> - - - <h3 id="ranking.matchPhase.maxHits">ranking.matchPhase.maxHits</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>long</td></tr> - <tr><td>Default</td><td>If sorting and not ranking: max(10000, maxhits+maxoffset). - Otherwise: <em>none</em>.</td></tr> - </table> - <p> - The max hits the engine should attempt to produce in the match phase on each partition. - If it is determined during matching that many more hits than this will be generated, the matching will fall back to - take the best (highest or lowest) values of the attribute given by ranking.matchPhase.attribute. - </p><p> - By default, this will be turned on only when sorting is used and grouping is not. - If sorting is used, the primary sort attribute will be used as the match phase attribute if it has fast-search set. - In that case the default can be overridden by setting this value explicitly. - </p> - - - <h3 id="ranking.matchPhase.attribute">ranking.matchPhase.attribute</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>An attribute name</td></tr> - <tr><td>Default</td><td><em>none</em></td></tr> - </table> - <p> - The attribute to decide which documents are a match if the match phase - estimates that there will be more than maxHits matches. - This attribute should have fast-search set and should correlate with the order - which would be produced by a full evaluation. - </p> - - - <h3 id="ranking.matchPhase.ascending">ranking.matchPhase.ascending</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>boolean</td></tr> - <tr><td>Default</td><td>false</td></tr> - </table> - <p> - Whether the attribute should be sorted in ascending or descending (default) order - to determine which documents to keep as matches. - </p> - - - <h3 id="ranking.matchPhase.diversity.attribute">ranking.matchPhase.diversity.attribute</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>An attribute name</td></tr> - <tr><td>Default</td><td>none.</td></tr> - </table> - <p> - The attribute to be used for producing the desired diversity. - Also see <a href="search-definitions-reference.html#diversity-attribute">attribute</a>. - </p> - - - <h3 id="ranking.matchPhase.diversity.minGroups">ranking.matchPhase.diversity.minGroups</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>long</td></tr> - <tr><td>Default</td><td>none</td></tr> - </table> - <p> - The minimum number of groups that should be returned from the match phase grouped by the diversity attribute. - Also see <a href="search-definitions-reference.html#diversity-min-groups">min-groups</a>. - </p> - - - - <h2 id="presentation">Presentation</h2> - - <h3 id="presentation.bolding">presentation.bolding [<em>bolding</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>bolding</td></tr> - <tr><td>Values</td><td>boolean</td></tr> - <tr><td>Default</td><td>true</td></tr> - </table> - <p> - Whether or not to bold search terms in <a href="search-definitions-reference.html">search definition</a> - fields defined with <a href="search-definitions-reference.html#bolding">bolding: on</a> - or <a href="search-definitions-reference.html#summary">summary: dynamic</a>. - </p> - - - <h3 id="presentation.format">presentation.format [<em>format</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>format</td></tr> - <tr><td>Values</td><td> - <table class="table table-striped"> - <tr> - <td><em>No value</em> or <code><a href="default-result-format.html">default</a></code></td> - <td>The default, builtin JSON format</td> - </tr> - <tr> - <td><code><a href="default-result-format.html">json</a></code></td> - <td>Builtin JSON format</td> - </tr> - <tr> - <td><code>xml</code></td> - <td>Deprecated, builtin XML format</td> - </tr> - <tr> - <td><code><a href="page-result-format.html">page</a></code></td> - <td>Alternative deprecated XML format which is suitable for use with <a href="../page-templates.html">page templates</a>.</td> - </tr> - <tr> - <td><em>Any other value</em></td> - <td>A custom <a href="../result-rendering.html">result renderer</a> supplied by the application - </tr> - </table> - - </td></tr> - <tr><td>Default</td><td>default</td></tr> - </table> - - - <h3 id="presentation.summary">presentation.summary [<em>summary</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>summary</td></tr> - <tr><td>Values</td><td> - The name of the <a href="../document-summaries.html#summary-classes-in-queries">summary class</a> - used to select fields in results. - </td></tr> - <tr><td>Default</td><td>The default summary class of the search definition.</td></tr> - </table> - - - <h3 id="presentation.template">presentation.template</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Any id specification of a deployed page template.</td></tr> - <tr><td>Default</td><td></td></tr> - </table> - <p> - The id of the page template to use for this result. This should be used with the - <a href="page-result-format.html">page</a> result format. - </p> - - - <h3 id="presentation.timing">presentation.timing</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>boolean</td></tr> - <tr><td>Default</td><td>false</td></tr> - </table> - <p> - Whether a result renderer should try to add optional timing information - to the rendered page. - </p> - - - - <h2 id="">Grouping and Aggregation</h2> - - <h3 id="select">select</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>A valid grouping specification.</td></tr> - <tr><td>Default</td><td>No grouping</td></tr> - </table> - <p> - Requests specific multi-level result set statistics and/or hit groups to be returned in the result. - Fields you want to retrieve statistics or hit groups for must be stored as document attributes - in the index structure by adding attribute to the indexing statement. - See the <a href="../grouping.html">grouping guide</a>. - </p> - - - <h3 id="collapsefield">collapsefield</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Any document summary field name</td></tr> - <tr><td>Default</td><td>No field collapsing</td></tr> - </table> - <p> - Collapse (i.e. aggregate) results using this field. - Collapsing is run in the container, not content node level. - Define a <em>collapsefield</em> to remove duplicates if the corpus has few duplicates - - this is more efficient than using <a href="#select">grouping</a>. - Otherwise, use grouping. - </p> - - - <h3 id="collapsesize">collapsesize</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>A positive integer</td></tr> - <tr><td>Default</td><td>1</td></tr> - </table> - <p>The number of hits to keep in each collapsed bucket</p> - - - <h3 id="collapse.summary">collapse.summary</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>A valid name of a document summary class.</td></tr> - <tr><td>Default</td><td>Use default summary or attributes.</td></tr> - </table> - <p>Use this summary class to fetch the field used for collapsing.</p> - - - - <h2 id="geographical-searches">Geographical Searches</h2> - - <h3 id="pos.ll">pos.ll</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td>Position given in latitude and longitude - example: <em>S22.4532;W123.9887</em> - Refer to <a href="search-definitions-reference.html#type:position">position field</a> - for format specification.</td> - </tr> - <tr><td>Default</td><td>None</td></tr> - </table> - - - <h3 id="pos.radius">pos.radius</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td> - Radius of the circle used for filtering. Valid units of measurement are km, m and mi. Examples: - <ul> - <li>pos.radius=100m</li> - <li>pos.radius=42mi</li> - <li>pos.radius=4km</li> - </ul> - One can also specify just a number (internal units, micro-degrees), but this is not recommended. - </td></tr> - <tr><td>Default</td><td>50km</td></tr> - </table> - - - <h3 id="pos.bb">pos.bb</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td> - Bounding box for positions, given as latitude and longitude boundaries. - The four boundaries must be specified as N, S, E, W, with degrees as - a decimal fraction. Degrees south of equator or west of Greenwich are - input as negative numbers. Examples: - <ul> - <li>n=37.44899,s=37.3323,e=-121.98241,w=-122.06566</li> - <li>s=40.183868,w=-74.819519,n=40.248291,e=-74.728798</li> - </ul> - </td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - - - <h3 id="pos.attribute">pos.attribute</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Any attribute that has zcurve encoded positions as a long attribute.</td></tr> - <tr><td>Default</td><td>Random choice among the ones declared as position in the searchdefinition.</td></tr> - </table> - <p> - Which attribute to use for the position. Can be both single- or multi-value. - </p> - - - - <h2 id="">Streaming Search</h2> - <p> - The features in this section applies to <a href="../streaming-search.html">streaming search</a> only. - </p> - - <h3 id="streaming.userid">streaming.userid</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>An integer in decimal notation in the range [0, 2^64></td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - Restricts streaming search to only stream through documents with document ids having the n=<number> - modifier and the userid part matches the supplied value. This can be used for grouping documents on a 64 bit integer. - </p> - - - <h3 id="streaming.groupname">streaming.groupname</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>A string</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - Restricts streaming search to only stream through documents with document ids having the g=<groupname> - modifier and the groupname part matches the supplied value. This can be used for grouping documents on a string. - </p> - - - <h3 id="streaming.selection">streaming.selection</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>A string</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - Restricts streaming search using a <a href="document-select-language.html">document selection</a>. - This can be used for selecting a subset of documents based on an advanced expression. - </p> - - - <h3 id="streaming.priority">streaming.priority</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td><a href="services.html#load-types">Priority class</a></td></tr> - <tr><td>Default</td><td>VERY_HIGH</td></tr> - </table> - <p> - Priority of the streaming search visitor. Having a high priority visitor helps maintain low latencies - even when the system is under load. - </p> - - - <h3 id="streaming.maxbucketspervisitor">streaming.maxbucketspervisitor</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>int</td></tr> - <tr><td>Default</td><td>1 (if ordering is set), or infinite</td></tr> - </table> - <p> - If set, visit only this many buckets at a time. - Combine with ordering to reduce visiting time for large users/groups. - </p> - - - - <h2 id="semantic-rules">Semantic Rules</h2> - <p> - Refer to <a href="semantic-rules.html">semantic rules</a>. - </p> - - <h3 id="rules.off">rules.off</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Boolean</td></tr> - <tr><td>Default</td><td>True</td></tr> - </table> - <p>Turn rule evaluation off for this query</p> - - - <h3 id="rules.rulebase">rules.rulebase</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>String</td></tr> - <tr><td>Default</td><td>A rule base name</td></tr> - </table> - <p>The name of the rule base to use for these queries</p> - - - <h3 id="tracelevel.rules">tracelevel.rules</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>int</td></tr> - <tr><td>Default</td><td>1-5 (?)</td></tr> - </table> - <p> - The amount of rule evaluation trace output to show, higher number means more details. - This is useful to see a trace from rule evaluation - without having to see trace from all other searchers at the same time. - </p> - - - - <h2 id="other">Other</h2> - - <h3 id="recall">recall</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Any allowed collection of recall terms</td></tr> - <tr><td>Default</td><td>No recall</td></tr> - </table> - <p> - Sets a recall parameter to be combined with the query. - This is identical to <a href="#model.filter">filter</a>, - except that recall terms are not exposed to the ranking framework and thus not ranked. - As such, one can not use unprefixed terms; they must either by positive or negative. - </p> - - - <h3 id="user">user</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>A string</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - The id of the user making the query. The contents of the argument are made available to the search chain, - but it triggers no features in Vespa apart from being propagated to the access log. - </p> - - - <h3 id="nocachewrite">nocachewrite</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Boolean</td></tr> - <tr><td>Default</td><td>False</td></tr> - </table> - <p>Set to true to avoid the result being written to cache when fetched.</p> - - - <h3 id="hitcountestimate">hitcountestimate</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Boolean</td></tr> - <tr><td>Default</td><td>False</td></tr> - </table> - <p>Make this an estimation query. - No hits will be returned, and total hit count will be set to an estimate of what executing - the query as a normal query would give. - </p> - - <h3 id="metrics.ignore">metrics.ignore</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Boolean</td></tr> - <tr><td>Default</td><td>False</td></tr> - </table> - <p>Ignore metric collection for this query request, useful for warm up queries</p> - - - </div> - - </div> -</div> - -</body> -</html> diff --git a/container-search-gui/src/main/resources/gui/icons/android-chrome-192x192.png b/container-search-gui/src/main/resources/gui/icons/android-chrome-192x192.png Binary files differindex af80e645b77..2e2d8952f86 100644 --- a/container-search-gui/src/main/resources/gui/icons/android-chrome-192x192.png +++ b/container-search-gui/src/main/resources/gui/icons/android-chrome-192x192.png diff --git a/container-search-gui/src/main/resources/gui/icons/android-chrome-512x512.png b/container-search-gui/src/main/resources/gui/icons/android-chrome-512x512.png Binary files differnew file mode 100644 index 00000000000..14724b65b30 --- /dev/null +++ b/container-search-gui/src/main/resources/gui/icons/android-chrome-512x512.png diff --git a/container-search-gui/src/main/resources/gui/icons/apple-touch-icon.png b/container-search-gui/src/main/resources/gui/icons/apple-touch-icon.png Binary files differindex 4dfc9a85a2a..402b914a17a 100644 --- a/container-search-gui/src/main/resources/gui/icons/apple-touch-icon.png +++ b/container-search-gui/src/main/resources/gui/icons/apple-touch-icon.png diff --git a/container-search-gui/src/main/resources/gui/icons/browserconfig.xml b/container-search-gui/src/main/resources/gui/icons/browserconfig.xml index efa70aac49b..6a1d00efa0e 100644 --- a/container-search-gui/src/main/resources/gui/icons/browserconfig.xml +++ b/container-search-gui/src/main/resources/gui/icons/browserconfig.xml @@ -1,10 +1,9 @@ <?xml version="1.0" encoding="utf-8"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <browserconfig> <msapplication> <tile> <square150x150logo src="/icons/mstile-150x150.png"/> - <TileColor>#2b5797</TileColor> + <TileColor>#da532c</TileColor> </tile> </msapplication> </browserconfig> diff --git a/container-search-gui/src/main/resources/gui/icons/favicon-16x16.png b/container-search-gui/src/main/resources/gui/icons/favicon-16x16.png Binary files differindex cce56472694..5cc83a38dba 100644 --- a/container-search-gui/src/main/resources/gui/icons/favicon-16x16.png +++ b/container-search-gui/src/main/resources/gui/icons/favicon-16x16.png diff --git a/container-search-gui/src/main/resources/gui/icons/favicon-32x32.png b/container-search-gui/src/main/resources/gui/icons/favicon-32x32.png Binary files differindex 5f9f3e1b85b..dfe72c4aa0a 100644 --- a/container-search-gui/src/main/resources/gui/icons/favicon-32x32.png +++ b/container-search-gui/src/main/resources/gui/icons/favicon-32x32.png diff --git a/container-search-gui/src/main/resources/gui/icons/favicon.ico b/container-search-gui/src/main/resources/gui/icons/favicon.ico Binary files differindex 885fb77541d..52564baa44d 100644 --- a/container-search-gui/src/main/resources/gui/icons/favicon.ico +++ b/container-search-gui/src/main/resources/gui/icons/favicon.ico diff --git a/container-search-gui/src/main/resources/gui/icons/manifest.json b/container-search-gui/src/main/resources/gui/icons/manifest.json index f71a0007cf4..5b2f3618461 100644 --- a/container-search-gui/src/main/resources/gui/icons/manifest.json +++ b/container-search-gui/src/main/resources/gui/icons/manifest.json @@ -1,5 +1,6 @@ { "name": "", + "short_name": "", "icons": [ { "src": "/icons/android-chrome-192x192.png", @@ -7,12 +8,12 @@ "type": "image/png" }, { - "src": "/icons/android-chrome-384x384.png", - "sizes": "384x384", + "src": "/icons/android-chrome-512x512.png", + "sizes": "512x512", "type": "image/png" } ], "theme_color": "#ffffff", "background_color": "#ffffff", "display": "standalone" -}
\ No newline at end of file +} diff --git a/container-search-gui/src/main/resources/gui/icons/mstile-144x144.png b/container-search-gui/src/main/resources/gui/icons/mstile-144x144.png Binary files differnew file mode 100644 index 00000000000..9eacb6051d8 --- /dev/null +++ b/container-search-gui/src/main/resources/gui/icons/mstile-144x144.png diff --git a/container-search-gui/src/main/resources/gui/icons/mstile-150x150.png b/container-search-gui/src/main/resources/gui/icons/mstile-150x150.png Binary files differindex 15ba5c1404e..736a5ca47d4 100644 --- a/container-search-gui/src/main/resources/gui/icons/mstile-150x150.png +++ b/container-search-gui/src/main/resources/gui/icons/mstile-150x150.png diff --git a/container-search-gui/src/main/resources/gui/icons/mstile-310x150.png b/container-search-gui/src/main/resources/gui/icons/mstile-310x150.png Binary files differnew file mode 100644 index 00000000000..0542b761ba4 --- /dev/null +++ b/container-search-gui/src/main/resources/gui/icons/mstile-310x150.png diff --git a/container-search-gui/src/main/resources/gui/icons/mstile-310x310.png b/container-search-gui/src/main/resources/gui/icons/mstile-310x310.png Binary files differnew file mode 100644 index 00000000000..5fc7990af48 --- /dev/null +++ b/container-search-gui/src/main/resources/gui/icons/mstile-310x310.png diff --git a/container-search-gui/src/main/resources/gui/icons/mstile-70x70.png b/container-search-gui/src/main/resources/gui/icons/mstile-70x70.png Binary files differnew file mode 100644 index 00000000000..4064cb227c4 --- /dev/null +++ b/container-search-gui/src/main/resources/gui/icons/mstile-70x70.png diff --git a/container-search-gui/src/main/resources/gui/icons/safari-pinned-tab.svg b/container-search-gui/src/main/resources/gui/icons/safari-pinned-tab.svg index 7b775fcdbde..5589b2f5ed6 100644 --- a/container-search-gui/src/main/resources/gui/icons/safari-pinned-tab.svg +++ b/container-search-gui/src/main/resources/gui/icons/safari-pinned-tab.svg @@ -2,21 +2,23 @@ <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd"> <svg version="1.0" xmlns="http://www.w3.org/2000/svg" - width="464.000000pt" height="464.000000pt" viewBox="0 0 464.000000 464.000000" + width="700.000000pt" height="700.000000pt" viewBox="0 0 700.000000 700.000000" preserveAspectRatio="xMidYMid meet"> <metadata> -Created by potrace 1.11, written by Peter Selinger 2001-2013 +Created by potrace 1.14, written by Peter Selinger 2001-2017 </metadata> -<g transform="translate(0.000000,464.000000) scale(0.100000,-0.100000)" +<g transform="translate(0.000000,700.000000) scale(0.100000,-0.100000)" fill="#000000" stroke="none"> -<path d="M2559 4542 c-129 -88 -531 -360 -599 -406 -25 -17 -49 -34 -55 -38 --5 -5 -26 -19 -45 -31 -35 -23 -183 -124 -761 -517 -68 -46 -144 -97 -169 --114 -25 -16 -46 -31 -48 -32 -2 -1 -4 -126 -4 -278 l1 -276 -27 18 c-15 11 --148 104 -297 209 -148 104 -294 206 -323 226 l-52 37 0 -1052 1 -1053 627 --425 c919 -624 937 -636 1056 -717 l109 -75 196 133 c603 409 710 482 1061 -721 146 99 327 221 402 271 l137 92 3 277 3 277 285 -201 c157 -110 313 -220 -348 -245 l62 -44 0 1053 -1 1053 -92 62 c-50 34 -144 97 -207 140 -63 43 -214 -146 -335 228 -121 82 -431 292 -688 467 -258 175 -469 318 -471 318 -1 0 -54 --35 -117 -78z"/> +<path d="M3820 6438 c-74 -50 -189 -128 -255 -173 -66 -45 -203 -138 -305 +-207 -325 -220 -883 -599 -1220 -828 -77 -52 -209 -142 -295 -200 l-155 -105 +-2 -363 -3 -362 -454 320 c-250 176 -455 320 -457 320 -2 0 -4 -623 -4 -1384 +l0 -1384 262 -178 c145 -98 304 -205 353 -239 50 -34 169 -115 265 -180 96 +-65 216 -146 265 -180 84 -57 163 -111 645 -438 102 -70 271 -185 377 -257 +105 -71 194 -130 198 -130 6 0 161 104 530 355 66 45 212 145 325 221 113 76 +315 213 448 304 133 91 322 219 420 285 97 66 247 168 332 226 85 58 191 130 +235 160 l80 54 0 362 c0 200 1 363 3 363 4 0 45 -29 503 -352 207 -147 386 +-271 398 -277 l21 -12 0 1384 0 1383 -247 168 c-137 92 -369 250 -518 351 +-148 101 -394 267 -545 370 -151 103 -358 243 -460 312 -102 69 -268 182 -370 +251 -102 69 -192 131 -201 139 -8 7 -20 13 -25 13 -5 0 -70 -41 -144 -92z"/> </g> </svg> diff --git a/container-search-gui/src/main/resources/gui/img/Vespa-V2.png b/container-search-gui/src/main/resources/gui/img/Vespa-V2.png Binary files differdeleted file mode 100644 index ac87f8e94d0..00000000000 --- a/container-search-gui/src/main/resources/gui/img/Vespa-V2.png +++ /dev/null diff --git a/container-search-gui/src/main/resources/gui/img/VespaIcon.png b/container-search-gui/src/main/resources/gui/img/VespaIcon.png Binary files differdeleted file mode 100644 index 33063432c20..00000000000 --- a/container-search-gui/src/main/resources/gui/img/VespaIcon.png +++ /dev/null diff --git a/container-search-gui/src/main/resources/gui/img/information.svg b/container-search-gui/src/main/resources/gui/img/information.svg deleted file mode 100644 index da42cf2caf6..00000000000 --- a/container-search-gui/src/main/resources/gui/img/information.svg +++ /dev/null @@ -1,10 +0,0 @@ -<?xml version="1.0"?> -<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" id="Capa_1" x="0px" y="0px" viewBox="0 0 512 512" style="enable-background:new 0 0 512 512;" xml:space="preserve" width="512px" height="512px"><g><g> - <g> - <g> - <circle cx="256" cy="378.5" r="25" data-original="#000000" class="active-path" data-old_color="#898989" fill="#767474"/> - <path d="M256,0C114.516,0,0,114.497,0,256c0,141.484,114.497,256,256,256c141.484,0,256-114.497,256-256 C512,114.516,397.503,0,256,0z M256,472c-119.377,0-216-96.607-216-216c0-119.377,96.607-216,216-216 c119.377,0,216,96.607,216,216C472,375.377,375.393,472,256,472z" data-original="#000000" class="active-path" data-old_color="#898989" fill="#767474"/> - <path d="M256,128.5c-44.112,0-80,35.888-80,80c0,11.046,8.954,20,20,20s20-8.954,20-20c0-22.056,17.944-40,40-40 c22.056,0,40,17.944,40,40c0,22.056-17.944,40-40,40c-11.046,0-20,8.954-20,20v50c0,11.046,8.954,20,20,20 c11.046,0,20-8.954,20-20v-32.531c34.466-8.903,60-40.26,60-77.469C336,164.388,300.112,128.5,256,128.5z" data-original="#000000" class="active-path" data-old_color="#898989" fill="#767474"/> - </g> - </g> -</g></g> </svg> diff --git a/container-search-gui/src/main/resources/gui/img/reload.svg b/container-search-gui/src/main/resources/gui/img/reload.svg deleted file mode 100644 index c5381f9f232..00000000000 --- a/container-search-gui/src/main/resources/gui/img/reload.svg +++ /dev/null @@ -1,6 +0,0 @@ -<?xml version="1.0"?> -<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" id="Layer_1" x="0px" y="0px" viewBox="0 0 512 512" style="enable-background:new 0 0 512 512;" xml:space="preserve" width="512px" height="512px" class=""><g><g> - <g> - <path d="M482.195,226.196C482.195,101.471,380.725,0,256.001,0S29.805,101.471,29.805,226.196c0,7.409,6.007,13.416,13.416,13.416 s13.416-6.008,13.416-13.416c0-109.93,89.434-199.363,199.363-199.363s199.363,89.434,199.363,199.363 c0,109.928-89.434,199.362-199.363,199.362h-23.276l33.282-37.255c4.937-5.525,4.458-14.007-1.067-18.944 c-5.525-4.937-14.008-4.457-18.944,1.068l-47.576,53.255c-7.788,8.718-7.788,21.866,0,30.584l47.576,53.255 c2.651,2.968,6.322,4.478,10.01,4.478c3.181,0,6.375-1.126,8.934-3.41c5.526-4.937,6.004-13.419,1.067-18.944l-33.282-37.255 h23.276C380.725,452.39,482.195,350.919,482.195,226.196z" data-original="#000000" class="active-path" data-old_color="#F0EDED" fill="#F3F2F2"/> - </g> -</g></g> </svg> diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 7f100df4e2c..f53856d465a 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -7974,6 +7974,7 @@ ], "methods": [ "public void <init>(java.lang.String)", + "public com.yahoo.search.schema.DocumentSummary$Builder addField(java.lang.String, java.lang.String)", "public com.yahoo.search.schema.DocumentSummary$Builder add(com.yahoo.search.schema.DocumentSummary$Field)", "public com.yahoo.search.schema.DocumentSummary$Builder setDynamic(boolean)", "public com.yahoo.search.schema.DocumentSummary build()" @@ -8040,7 +8041,7 @@ ], "methods": [ "public java.lang.String name()", - "public java.util.List fields()", + "public java.util.Map fields()", "public boolean isDynamic()", "public boolean equals(java.lang.Object)", "public int hashCode()", diff --git a/container-search/pom.xml b/container-search/pom.xml index ccec1330d68..031d25c7d02 100644 --- a/container-search/pom.xml +++ b/container-search/pom.xml @@ -73,6 +73,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-disc</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>xerces</groupId> <artifactId>xercesImpl</artifactId> <exclusions> diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index 33b912da42f..a9a36d40f93 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -25,7 +25,7 @@ public class DocsumDefinition { public DocsumDefinition(DocumentSummary documentSummary) { this.name = documentSummary.name(); this.dynamic = documentSummary.isDynamic(); - this.fields = documentSummary.fields() + this.fields = documentSummary.fields().values() .stream() .map(field -> DocsumField.create(field.name(), field.type().asString())) .collect(Collectors.toUnmodifiableMap(field -> field.getName(), diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 48885e4b3da..3a2bccf017e 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -438,7 +438,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { for (FieldDescription field : arguments.fields().values()) { if (field.getType() == FieldType.genericQueryProfileType) { // Generic map - CompoundName fullName = prefix.append(field.getName()); + CompoundName fullName = prefix.append(field.getCompoundName()); for (Map.Entry<String, Object> entry : originalProperties.listProperties(fullName, context).entrySet()) { properties().set(fullName.append(entry.getKey()), entry.getValue(), context); } @@ -447,7 +447,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { setFrom(prefix, originalProperties, ((QueryProfileFieldType)field.getType()).getQueryProfileType(), context); } else { - CompoundName fullName = prefix.append(field.getName()); + CompoundName fullName = prefix.append(field.getCompoundName()); Object value = originalProperties.get(fullName, context); if (value != null) { properties().set(fullName, value, context); diff --git a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java new file mode 100644 index 00000000000..5b8c99506c5 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java @@ -0,0 +1,81 @@ +package com.yahoo.search.handler; + +import com.fasterxml.jackson.core.JsonLocation; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.processing.IllegalInputException; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +/** + * Parser that consumes json and creates a single level key value map by dotting nested objects. + * This is specially tailored for the json query input coming as post. + * It does the cheapest possible json parsing delaying number parsing to where it is needed and avoids dreaded toString() + * of complicated json object trees. + * + * @author baldersheim + */ +class Json2SingleLevelMap { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private final byte [] buf; + private final JsonParser parser; + Json2SingleLevelMap(InputStream data) { + try { + buf = data.readAllBytes(); + parser = jsonMapper.createParser(buf); + } catch (IOException e) { + throw new RuntimeException("Problem reading POSTed data", e); + } + } + Map<String, String> parse() { + try { + Map<String, String> map = new HashMap<>(); + if (parser.nextToken() != JsonToken.START_OBJECT) { + throw new IllegalInputException("Expected start of object, got '" + parser.currentToken() + "'"); + } + parse(map, ""); + return map; + } catch (JsonParseException e) { + throw new IllegalInputException("Json parse error.", e); + } catch (IOException e) { + throw new RuntimeException("Problem reading POSTed data", e); + } + } + void parse(Map<String, String> map, String parent) throws IOException { + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parent + parser.getCurrentName(); + JsonToken token = parser.nextToken(); + if ((token == JsonToken.VALUE_STRING) || + (token == JsonToken.VALUE_NUMBER_FLOAT) || + (token == JsonToken.VALUE_NUMBER_INT) || + (token == JsonToken.VALUE_TRUE) || + (token == JsonToken.VALUE_FALSE) || + (token == JsonToken.VALUE_NULL)) { + map.put(fieldName, parser.getText()); + } else if (token == JsonToken.START_ARRAY) { + map.put(fieldName, skipChildren(parser, buf)); + } else if (token == JsonToken.START_OBJECT) { + if (fieldName.equals("select.where") || fieldName.equals("select.grouping")) { + map.put(fieldName, skipChildren(parser, buf)); + } else { + parse(map, fieldName + "."); + } + } else { + throw new IllegalInputException("In field '" + fieldName + "', got unknown json token '" + token.asString() + "'"); + } + } + } + private String skipChildren(JsonParser parser, byte [] input) throws IOException { + JsonLocation start = parser.getCurrentLocation(); + parser.skipChildren(); + JsonLocation end = parser.getCurrentLocation(); + int offset = (int)start.getByteOffset() - 1; + return new String(input, offset, (int)(end.getByteOffset() - offset), StandardCharsets.UTF_8); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 2c26a302ece..76702a1d4e0 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -17,7 +17,6 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.jdisc.RequestHandlerSpec; import com.yahoo.container.jdisc.VespaHeaders; -import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; import com.yahoo.language.process.Embedder; @@ -43,14 +42,13 @@ import com.yahoo.search.searchchain.SearchChainRegistry; import com.yahoo.search.statistics.ElapsedTime; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.SlimeUtils; import com.yahoo.yolean.Exceptions; import com.yahoo.yolean.trace.TraceNode; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.Executor; @@ -490,23 +488,9 @@ public class SearchHandler extends LoggingRequestHandler { || ! JSON_CONTENT_TYPE.equals(getMediaType(request))) return request.propertyMap(); - Inspector inspector; - try { - // Use an 4k buffer, that should be plenty for most json requests to pass in a single chunk - byte[] byteArray = IOUtils.readBytes(request.getData(), 4096); - inspector = SlimeUtils.jsonToSlime(byteArray).get(); - if (inspector.field("error_message").valid()) { - throw new IllegalInputException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + - new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); - } - - } catch (IOException e) { - throw new RuntimeException("Problem reading POSTed data", e); - } + Map<String, String> requestMap = new Json2SingleLevelMap(request.getData()).parse(); // Add fields from JSON to the request map - Map<String, String> requestMap = new HashMap<>(); - createRequestMapping(inspector, requestMap, ""); requestMap.putAll(request.propertyMap()); if (requestMap.containsKey("yql") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) @@ -517,35 +501,13 @@ public class SearchHandler extends LoggingRequestHandler { return requestMap; } + @Deprecated // TODO: Remove on Vespa 9 public void createRequestMapping(Inspector inspector, Map<String, String> map, String parent) { - inspector.traverse((ObjectTraverser) (key, value) -> { - String qualifiedKey = parent + key; - switch (value.type()) { - case BOOL: - map.put(qualifiedKey, Boolean.toString(value.asBool())); - break; - case DOUBLE: - map.put(qualifiedKey, Double.toString(value.asDouble())); - break; - case LONG: - map.put(qualifiedKey, Long.toString(value.asLong())); - break; - case STRING: - map.put(qualifiedKey , value.asString()); - break; - case ARRAY: - map.put(qualifiedKey, value.toString()); // XXX: Causes parsing the JSON twice (Query.setPropertiesFromRequestMap) - break; - case OBJECT: - if (qualifiedKey.equals("select.where") || qualifiedKey.equals("select.grouping")) { - map.put(qualifiedKey, value.toString()); // XXX: Causes parsing the JSON twice (Query.setPropertiesFromRequestMap) - break; - } - createRequestMapping(value, map, qualifiedKey + "."); - break; - } - - }); + try { + new Json2SingleLevelMap(new ByteArrayInputStream(inspector.toString().getBytes(StandardCharsets.UTF_8))).parse(map, parent); + } catch (IOException e) { + throw new RuntimeException("Failed creating request mapping for parent '" + parent + "'", e); + } } @Override diff --git a/container-search/src/main/java/com/yahoo/search/handler/observability/SearchStatusExtension.java b/container-search/src/main/java/com/yahoo/search/handler/observability/SearchStatusExtension.java new file mode 100644 index 00000000000..836bb1b8354 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/handler/observability/SearchStatusExtension.java @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.handler.observability; + +import com.fasterxml.jackson.databind.JsonNode; +import com.yahoo.container.handler.observability.ApplicationStatusHandler; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.search.handler.SearchHandler; +import com.yahoo.search.searchchain.SearchChainRegistry; + +import java.util.Map; + +/** + * @author bjorncs + */ +public class SearchStatusExtension implements ApplicationStatusHandler.Extension { + + @Override + public Map<String, ? extends JsonNode> produceExtraFields(ApplicationStatusHandler statusHandler) { + return Map.of("searchChains", renderSearchChains(statusHandler)); + } + + private static JsonNode renderSearchChains(ApplicationStatusHandler statusHandler) { + for (RequestHandler h : statusHandler.requestHandlers()) { + if (h instanceof SearchHandler) { + SearchChainRegistry scReg = ((SearchHandler) h).getSearchChainRegistry(); + return ApplicationStatusHandler.renderChains(scReg); + } + } + return statusHandler.jsonMapper().createObjectNode(); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/handler/observability/package-info.java b/container-search/src/main/java/com/yahoo/search/handler/observability/package-info.java new file mode 100644 index 00000000000..baf24b4a94d --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/handler/observability/package-info.java @@ -0,0 +1,8 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.search.handler.observability; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 8c1a0ac1d25..7f5df94d020 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -31,9 +31,13 @@ import java.util.Map; */ public class QueryProfileProperties extends Properties { + private static final String ENVIRONMENT = "environment"; + private static final String REGION = "region"; + private static final String INSTANCE = "instance"; private final CompiledQueryProfile profile; private final Map<String, Embedder> embedders; private final ZoneInfo zoneInfo; + private final Map<String, String> zoneContext; // Note: The priority order is: values has precedence over references @@ -68,6 +72,11 @@ public class QueryProfileProperties extends Properties { this.profile = profile; this.embedders = embedders; this.zoneInfo = zoneInfo; + this.zoneContext = Map.of( + ENVIRONMENT, zoneInfo.zone().environment().name(), + REGION, zoneInfo.zone().region(), + INSTANCE, zoneInfo.application().instance()); + } /** Returns the query profile backing this, or null if none */ @@ -289,11 +298,14 @@ public class QueryProfileProperties extends Properties { private Map<String, String> contextWithZoneInfo(Map<String, String> context) { if (zoneInfo == ZoneInfo.defaultInfo()) return context; - - Map<String, String> contextWithZoneInfo = context == null ? new HashMap<>() : new HashMap<>(context); - contextWithZoneInfo.putIfAbsent("environment", zoneInfo.zone().environment().name()); - contextWithZoneInfo.putIfAbsent("region", zoneInfo.zone().region()); - contextWithZoneInfo.putIfAbsent("instance", zoneInfo.application().instance()); + if (context == null || context.isEmpty()) return zoneContext; + if (context == zoneContext) return context; + if (context.containsKey(ENVIRONMENT) && context.containsKey(REGION) && context.containsKey(INSTANCE)) return context; + + Map<String, String> contextWithZoneInfo = new HashMap<>(context); + contextWithZoneInfo.putIfAbsent(ENVIRONMENT, zoneInfo.zone().environment().name()); + contextWithZoneInfo.putIfAbsent(REGION, zoneInfo.zone().region()); + contextWithZoneInfo.putIfAbsent(INSTANCE, zoneInfo.application().instance()); return Collections.unmodifiableMap(contextWithZoneInfo); } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java index 72388f77173..f60d9820706 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.search.query.properties; -import com.yahoo.processing.request.*; +import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.Properties; import java.util.Map; import java.util.LinkedHashMap; diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index f6e158cf04a..160a039fe2d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -300,7 +300,7 @@ public class QueryProperties extends Properties { } else if (key.size() == 3 && key.get(1).equals(Presentation.FORMAT)) { if (key.last().equals(Presentation.TENSORS)) - query.getPresentation().setTensorShortForm(asString(value, "")); + query.getPresentation().setTensorShortForm(asString(value, "short")); else throwIllegalParameter(key.last(), Presentation.FORMAT); } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java index ad212d05780..0ac61246c48 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java @@ -1,12 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.properties; -import ai.vespa.cloud.ZoneInfo; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.Properties; import java.util.Collections; -import java.util.HashMap; import java.util.Map; /** diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 58353cc5907..4f55b9946a8 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -124,13 +124,14 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private volatile JsonGenerator generator; private volatile FieldConsumer fieldConsumer; private volatile Deque<Integer> renderedChildren; + static class FieldConsumerSettings { volatile boolean debugRendering = false; volatile boolean jsonDeepMaps = true; volatile boolean jsonWsets = true; volatile boolean jsonMapsAll = true; volatile boolean jsonWsetsAll = false; - volatile boolean tensorShortForm = false; + volatile boolean tensorShortForm = true; boolean convertDeep() { return (jsonDeepMaps || jsonWsets); } void init() { this.debugRendering = false; @@ -138,7 +139,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { this.jsonWsets = true; this.jsonMapsAll = true; this.jsonWsetsAll = true; - this.tensorShortForm = false; + this.tensorShortForm = true; } void getSettings(Query q) { if (q == null) { @@ -155,6 +156,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { this.tensorShortForm = q.getPresentation().getTensorShortForm(); } } + private volatile FieldConsumerSettings fieldConsumerSettings; private volatile LongSupplier timeSource; private volatile OutputStream stream; diff --git a/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java b/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java index 12037ee6633..0aec6b0a4f6 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java +++ b/container-search/src/main/java/com/yahoo/search/schema/DocumentSummary.java @@ -3,7 +3,10 @@ package com.yahoo.search.schema; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; /** @@ -15,17 +18,17 @@ import java.util.Objects; public class DocumentSummary { private final String name; - private final List<Field> fields; + private final Map<String, Field> fields; private final boolean dynamic; private DocumentSummary(Builder builder) { this.name = builder.name; - this.fields = List.copyOf(builder.fields); + this.fields = Collections.unmodifiableMap(builder.fields); this.dynamic = builder.dynamic; } public String name() { return name; } - public List<Field> fields() { return fields; } + public Map<String, Field> fields() { return fields; } /** Returns whether this contains fields which are generated dynamically from the query and field data. */ public boolean isDynamic() { return dynamic; } @@ -54,15 +57,20 @@ public class DocumentSummary { public static class Builder { private final String name; - private final List<Field> fields = new ArrayList<>(); + private final Map<String, Field> fields = new LinkedHashMap<>(); private boolean dynamic; public Builder(String name) { this.name = name; } + public Builder addField(String name, String type) { + fields.put(name, new Field(name, type)); + return this; + } + public Builder add(Field field) { - fields.add(field); + fields.put(field.name(), field); return this; } diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java index c5a74ece866..ed1d25a0b35 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java @@ -1,17 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.schema; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.yahoo.container.QrSearchersConfig; -import com.yahoo.prelude.fastsearch.DocsumDefinition; -import com.yahoo.prelude.fastsearch.DocsumField; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.tensor.TensorType; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index 2dfa6ef3e3a..db44e13c27e 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -1,104 +1,45 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.yql; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Set; -import com.yahoo.api.annotations.Beta; import com.yahoo.component.chain.dependencies.After; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass.Fields; import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.query.Presentation; +import com.yahoo.search.schema.DocumentSummary; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; /** - * Ensure the fields specified in {@link Presentation#getSummaryFields()} are - * available after filling phase. + * Ensure the fields specified in {@link Presentation#getSummaryFields()} are available after filling phase. * * @author stiankri * @author Steinar Knutsen */ -@Beta @After(MinimalQueryInserter.EXTERNAL_YQL) public class FieldFiller extends Searcher { private final Set<String> intersectionOfAttributes; - private final SummaryIntersections summaryDb = new SummaryIntersections(); + private final SchemaInfo schemaInfo; public static final CompoundName FIELD_FILLER_DISABLE = new CompoundName("FieldFiller.disable"); - private static class SummaryIntersections { - private final Map<String, Map<String, Set<String>>> db = new HashMap<>(); + public FieldFiller(SchemaInfo schemaInfo) { + this.schemaInfo = schemaInfo; - void add(String dbName, Summaryclass summary) { - Map<String, Set<String>> docType = getOrCreateDocType(dbName); - Set<String> fields = new HashSet<>(summary.fields().size()); - for (Fields f : summary.fields()) { - fields.add(f.name()); - } - docType.put(summary.name(), fields); - } - - private Map<String, Set<String>> getOrCreateDocType(String dbName) { - Map<String, Set<String>> docType = db.get(dbName); - if (docType == null) { - docType = new HashMap<>(); - db.put(dbName, docType); - } - return docType; - } - - boolean hasAll(Set<String> requested, String summaryName, Set<String> restrict) { - Set<String> explicitRestriction; - Set<String> intersection = null; - - if (restrict.isEmpty()) { - explicitRestriction = db.keySet(); - } else { - explicitRestriction = restrict; - } - - for (String docType : explicitRestriction) { - Map<String, Set<String>> summaries = db.get(docType); - Set<String> summary; - - if (summaries == null) { - continue; - } - summary = summaries.get(summaryName); - if (summary == null) { - intersection = null; - break; - } - if (intersection == null) { - intersection = new HashSet<>(summary.size()); - intersection.addAll(summary); - } else { - intersection.retainAll(summary); - } - } - return intersection != null && intersection.containsAll(requested); - } - } - - public FieldFiller(DocumentdbInfoConfig config) { intersectionOfAttributes = new HashSet<>(); boolean first = true; - - for (Documentdb db : config.documentdb()) { - for (Summaryclass summary : db.summaryclass()) { + for (Schema schema : schemaInfo.schemas().values()) { + for (DocumentSummary summary : schema.documentSummaries().values()) { Set<String> attributes; if (SORTABLE_ATTRIBUTES_SUMMARY_CLASS.equals(summary.name())) { attributes = new HashSet<>(summary.fields().size()); - for (Fields f : summary.fields()) { + for (DocumentSummary.Field f : summary.fields().values()) { attributes.add(f.name()); } if (first) { @@ -108,12 +49,6 @@ public class FieldFiller extends Searcher { intersectionOfAttributes.retainAll(attributes); } } - // yes, we store attribute prefetch here as well, this is in - // case we get a query where we have a restrict parameter which - // makes filling with attribute prefetch possible even though it - // wouldn't have been possible without restricting the set of - // doctypes - summaryDb.add(db.name(), summary); } } } @@ -140,10 +75,31 @@ public class FieldFiller extends Searcher { } } else { // Yes, summaryClass may be SORTABLE_ATTRIBUTES_SUMMARY_CLASS here - if ( ! summaryDb.hasAll(summaryFields, summaryClass, result.getQuery().getModel().getRestrict())) { + if ( ! hasAll(summaryFields, summaryClass, result.getQuery().getModel().getRestrict())) { execution.fill(result, null); } } } + private boolean hasAll(Set<String> requested, String summaryName, Set<String> restrict) { + Set<String> intersection = null; + for (String schemaName : restrict.isEmpty() ? schemaInfo.schemas().keySet() : restrict) { + Schema schema = schemaInfo.schemas().get(schemaName); + if (schema == null) continue; + + DocumentSummary summary = schema.documentSummaries().get(summaryName); + if (summary == null) { + intersection = null; + break; + } + if (intersection == null) { + intersection = new HashSet<>(summary.fields().size()); + intersection.addAll(summary.fields().keySet()); + } else { + intersection.retainAll(summary.fields().keySet()); + } + } + return intersection != null && intersection.containsAll(requested); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java index e1400e4f860..48c48748563 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java +++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java @@ -63,7 +63,7 @@ public class MinimalQueryInserter extends Searcher { Query query = new Query("search/?yql=select%20*%20from%20sources%20where%20title%20contains%20'xyz'"); Result result = insertQuery(query, new ParserEnvironment().setLinguistics(linguistics)); if (result != null) { - log.warning("Warmup code trigger an error. Error = " + result.toString()); + log.warning("Warmup code trigger an error. Error = " + result); return false; } if ( ! "select * from sources where title contains \"xyz\"".equals(query.yqlRepresentation())) { diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index fc0cef2fb5e..5258087eb44 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -914,7 +914,13 @@ final class ProgramParser { String text = literal.getChild(0).getText(); switch(parseTreeIndex) { case yqlplusParser.INT: - return Integer.valueOf(text); + Long as_long = Long.valueOf(text); + int as_int = as_long.intValue(); + if (as_int == as_long) { + return Integer.valueOf(as_int); + } else { + return as_long; + } case yqlplusParser.FLOAT: return Double.valueOf(text); case yqlplusParser.STRING: diff --git a/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def b/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def index 4528475697b..a16ef0ac8e7 100644 --- a/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def +++ b/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def @@ -1,26 +1,29 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=prelude.fastsearch +# Contains the names of the schams present in a particular content cluster, +# where the cluster in question is determined by the config id used to subscribe to this. + ## The name of the schema/document database documentdb[].name string -## The id of the summary class +## The id of the summary class. Not used TODO: Remove after July 2022 documentdb[].summaryclass[].id int -## The name of the summary class +## The name of the summary class. Not used TODO: Remove after July 2022 documentdb[].summaryclass[].name string -## The name of a field in the summary class +## The name of a field in the summary class. Not used TODO: Remove after July 2022 documentdb[].summaryclass[].fields[].name string -## The type of a field in the summary class +## The type of a field in the summary class. Not used TODO: Remove after July 2022 documentdb[].summaryclass[].fields[].type string -## Whether this field is a dynamic snippet +## Whether this field is a dynamic snippet. Not used TODO: Remove after July 2022 documentdb[].summaryclass[].fields[].dynamic bool default=false -## Information about rank profiles +## Information about rank profiles. Not used TODO: Remove after July 2022 documentdb[].rankprofile[].name string documentdb[].rankprofile[].hasSummaryFeatures bool default=true documentdb[].rankprofile[].hasRankFeatures bool default=true -# The name of an input (query rank feature) accepted by this profile +# The name of an input (query rank feature) accepted by this profile. Not used TODO: Remove after July 2022 documentdb[].rankprofile[].input[].name string -# The tensor type of an input (query rank feature) accepted by this profile +# The tensor type of an input (query rank feature) accepted by this profile. Not used TODO: Remove after July 2022 documentdb[].rankprofile[].input[].type string diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index a09c2ff9b79..ac860faffa5 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -127,11 +127,11 @@ public class ClusterSearcherTestCase { @Test public void testThatDocumentTypesAreResolvedTODO_REMOVE() { - ClusterSearcher cluster1 = new ClusterSearcher(new LinkedHashSet<>(Arrays.asList("type1", "type2", "type3"))); + ClusterSearcher cluster1 = new ClusterSearcher(new LinkedHashSet<>(List.of("type1", "type2", "type3"))); try { - ClusterSearcher type1 = new ClusterSearcher(new LinkedHashSet<>(Arrays.asList("type6"))); + ClusterSearcher type1 = new ClusterSearcher(new LinkedHashSet<>(List.of("type6"))); try { - assertEquals(new LinkedHashSet<>(Arrays.asList()), resolve(cluster1, "&sources=cluster2")); + assertEquals(new LinkedHashSet<>(List.of()), resolve(cluster1, "&sources=cluster2")); } finally { type1.deconstruct(); } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 271b932dff5..d5630063f4b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -214,7 +214,6 @@ public class FastSearcherTestCase { private DocumentdbInfoConfig documentdbInfoConfig(String schemaName) { var db = new DocumentdbInfoConfig.Documentdb.Builder().name(schemaName); - db.rankprofile(new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder().name("default")); return new DocumentdbInfoConfig.Builder().documentdb(db).build(); } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java index 6c66c8f4ca6..b3afd8c5f50 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java @@ -7,7 +7,6 @@ import com.yahoo.net.URI; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/JSONSearchHandlerTestCase.java index dd78cfefe28..b0b88bf8190 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/JSONSearchHandlerTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.handler.test; +package com.yahoo.search.handler; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -12,7 +12,6 @@ import com.yahoo.container.jdisc.RequestHandlerTestDriver; import com.yahoo.container.protect.Error; import com.yahoo.io.IOUtils; import com.yahoo.net.HostName; -import com.yahoo.search.handler.SearchHandler; import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase; import com.yahoo.slime.Inspector; import com.yahoo.slime.SlimeUtils; @@ -20,18 +19,21 @@ import com.yahoo.test.json.JsonTestHelper; import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.HashMap; import java.util.Map; +import java.util.Random; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; @@ -51,7 +53,6 @@ public class JSONSearchHandlerTestCase { private static final String selfHostname = HostName.getLocalhost(); private static String tempDir = ""; - private static String configId = null; private static final String uri = "http://localhost?"; private static final String JSON_CONTENT_TYPE = "application/json"; @@ -66,7 +67,7 @@ public class JSONSearchHandlerTestCase { public void startUp() throws IOException { File cfgDir = tempfolder.newFolder("SearchHandlerTestCase"); tempDir = cfgDir.getAbsolutePath(); - configId = "dir:" + tempDir; + String configId = "dir:" + tempDir; IOUtils.copyDirectory(new File(testDir), cfgDir, 1); // make configs active generateComponentsConfigForActive(); @@ -324,8 +325,7 @@ public class JSONSearchHandlerTestCase { json.set("select", select); Inspector inspector = SlimeUtils.jsonToSlime(json.toString().getBytes(StandardCharsets.UTF_8)).get(); - Map<String, String> map = new HashMap<>(); - searchHandler.createRequestMapping(inspector, map, ""); + Map<String, String> map = new Json2SingleLevelMap(new ByteArrayInputStream(inspector.toString().getBytes(StandardCharsets.UTF_8))).parse(); JsonNode processedWhere = jsonMapper.readTree(map.get("select.where")); JsonTestHelper.assertJsonEquals(where.toString(), processedWhere.toString()); @@ -510,8 +510,7 @@ public class JSONSearchHandlerTestCase { // Create mapping Inspector inspector = SlimeUtils.jsonToSlime(json.toString().getBytes(StandardCharsets.UTF_8)).get(); - Map<String, String> map = new HashMap<>(); - searchHandler.createRequestMapping(inspector, map, ""); + Map<String, String> map = new Json2SingleLevelMap(new ByteArrayInputStream(inspector.toString().getBytes(StandardCharsets.UTF_8))).parse(); // Create GET-request with same query String url = uri + "&model.sources=source1%2Csource2&select=_all&model.language=en&presentation.timing=false&pos.attribute=default&pos.radius=71234m&model.searchPath=node1&nocachewrite=false&ranking.matchPhase.maxHits=100&presentation.summary=none" + @@ -536,4 +535,27 @@ public class JSONSearchHandlerTestCase { assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), "Application/JSON; charset=utf-8"), jsonResult); } + private static String createBenchmarkRequest(int num) { + Random rand = new Random(); + StringBuilder sb = new StringBuilder("{\"yql\": \"select id from vectors where {targetHits:10, approximate:true}nearestNeighbor(vector,q);\", \"input.query(q)\":["); + sb.append(rand.nextDouble()); + for (int i=1; i < num; i++) { + sb.append(','); + sb.append(rand.nextDouble()); + } + sb.append("]}"); + return sb.toString(); + } + @Ignore + public void benchmarkJsonParsing() { + String request = createBenchmarkRequest(768); + for (int i=0; i < 10000; i++) { + RequestHandlerTestDriver.MockResponseHandler responseHandler = + driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, request, JSON_CONTENT_TYPE); + String response = responseHandler.readAll(); + assertEquals(200, responseHandler.getStatus()); + assertFalse(response.isEmpty()); + } + } + } diff --git a/container-search/src/test/java/com/yahoo/search/handler/Json2SinglelevelMapTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/Json2SinglelevelMapTestCase.java new file mode 100644 index 00000000000..d8db88323c7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/Json2SinglelevelMapTestCase.java @@ -0,0 +1,32 @@ +package com.yahoo.search.handler; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class Json2SinglelevelMapTestCase { + @Test + public void testDecodeString() { + Map<String, String> m = new Json2SingleLevelMap(new ByteArrayInputStream("{\"yql\":\"text\", \"f1\":7.3, \"i1\":7, \"t\":true, \"f\":false, \"n\":null, \"a\":[0.786, 0.193]}".getBytes(StandardCharsets.UTF_8))).parse(); + assertEquals(7, m.size()); + assertTrue(m.containsKey("yql")); + assertTrue(m.containsKey("f1")); + assertTrue(m.containsKey("i1")); + assertTrue(m.containsKey("t")); + assertTrue(m.containsKey("f")); + assertTrue(m.containsKey("n")); + assertTrue(m.containsKey("a")); + assertEquals("text", m.get("yql")); + assertEquals("7.3", m.get("f1")); + assertEquals("7", m.get("i1")); + assertEquals("true", m.get("t")); + assertEquals("false", m.get("f")); + assertEquals("null", m.get("n")); + assertEquals("[0.786, 0.193]", m.get("a")); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 83d04c4cc6d..7ddc087feb7 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -116,8 +116,10 @@ public class JsonRendererTestCase { @After public void deconstructClone() { - renderer.deconstruct(); - renderer = null; + if (renderer != null) { + renderer.deconstruct(); + renderer = null; + } } @AfterClass @@ -436,10 +438,7 @@ public class JsonRendererTestCase { e2.search(subQuery); subQuery.trace("yellow", 1); q.trace("marker", 1); - ByteArrayOutputStream bs = new ByteArrayOutputStream(); - CompletableFuture<Boolean> f = renderer.renderResponse(bs, r, execution, null); - assertTrue(f.get()); - String summary = Utf8.toString(bs.toByteArray()); + String summary = render(execution, r); assertEqualJson(expected, summary); } @@ -478,10 +477,7 @@ public class JsonRendererTestCase { execution.search(q); new Execution(new Chain<>(), execution.context()); - ByteArrayOutputStream bs = new ByteArrayOutputStream(); - CompletableFuture<Boolean> f = renderer.renderResponse(bs, r, execution, null); - assertTrue(f.get()); - String summary = Utf8.toString(bs.toByteArray()); + String summary = render(execution, r); ObjectMapper m = new ObjectMapper(); Map<String, Object> exp = m.readValue(expected, Map.class); @@ -1478,10 +1474,15 @@ public class JsonRendererTestCase { } private String render(Execution execution, Result r) throws InterruptedException, ExecutionException { - ByteArrayOutputStream bs = new ByteArrayOutputStream(); - CompletableFuture<Boolean> f = renderer.renderResponse(bs, r, execution, null); - assertTrue(f.get()); - return Utf8.toString(bs.toByteArray()); + if (renderer == null) createClone(); + try { + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + CompletableFuture<Boolean> f = renderer.renderResponse(bs, r, execution, null); + assertTrue(f.get()); + return Utf8.toString(bs.toByteArray()); + } finally { + deconstructClone(); + } } @SuppressWarnings("unchecked") diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java index 0385de5bebf..ee933a86578 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -5,19 +5,18 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; +import com.yahoo.search.schema.DocumentSummary; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; import org.junit.After; import org.junit.Before; import org.junit.Test; import com.yahoo.component.chain.Chain; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass.Fields; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -57,10 +56,7 @@ public class YqlFieldAndSourceTestCase { mockBackend.addSummaryClassByCopy(SORTABLE_ATTRIBUTES_SUMMARY_CLASS, Arrays.asList(FIELD2)); mockBackend.addSummaryClassByCopy(THIRD_OPTION, Arrays.asList(FIELD3)); - DocumentdbInfoConfig config = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder() - .documentdb(buildDocumentdbArray())); - - searchChain = new Chain<>(new FieldFiller(config), mockBackend); + searchChain = new Chain<>(new FieldFiller(schemaInfo()), mockBackend); context = Execution.Context.createContextStub(); execution = new Execution(searchChain, context); } @@ -75,29 +71,13 @@ public class YqlFieldAndSourceTestCase { return h; } - private List<Documentdb.Builder> buildDocumentdbArray() { - List<Documentdb.Builder> configArray = new ArrayList<>(1); - configArray.add(new Documentdb.Builder().summaryclass( - buildSummaryclassArray()).name("defaultsearchdefinition")); - - return configArray; - } - - private List<Summaryclass.Builder> buildSummaryclassArray() { - return Arrays.asList( - new Summaryclass.Builder() - .id(0) - .name(DEFAULT_SUMMARY_CLASS) - .fields(Arrays.asList(new Fields.Builder().name(FIELD1).type("string"), - new Fields.Builder().name(FIELD2).type("string"))), - new Summaryclass.Builder() - .id(1) - .name(SORTABLE_ATTRIBUTES_SUMMARY_CLASS) - .fields(Arrays.asList(new Fields.Builder().name(FIELD2).type("string"))), - new Summaryclass.Builder() - .id(2) - .name(THIRD_OPTION) - .fields(Arrays.asList(new Fields.Builder().name(FIELD3).type("string")))); + private SchemaInfo schemaInfo() { + var schema = new Schema.Builder("defaultsearchdefinition"); + schema.add(new DocumentSummary.Builder(DEFAULT_SUMMARY_CLASS).addField(FIELD1, "string") + .addField(FIELD2, "string").build()) + .add((new DocumentSummary.Builder(SORTABLE_ATTRIBUTES_SUMMARY_CLASS).addField(FIELD2, "string").build())) + .add((new DocumentSummary.Builder(THIRD_OPTION).addField(FIELD3, "string").build())); + return new SchemaInfo(List.of(schema.build()), Map.of()); } @After diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 03472408783..299a7ff9d30 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -745,6 +745,7 @@ public class YqlParserTestCase { @Test public void testLongNumberInSimpleExpression() { assertParse("select foo from bar where price = 8589934592L", "price:8589934592"); + assertParse("select foo from bar where price = 8589934592", "price:8589934592"); } @Test diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java index 835235a593c..3cf2a38982a 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java @@ -46,6 +46,7 @@ import static org.mockito.Mockito.when; * @author Ulf Carlin */ public class VdsStreamingSearcherTestCase { + public static final String USERDOC_ID_PREFIX = "id:namespace:mytype:n=1:userspecific"; public static final String GROUPDOC_ID_PREFIX = "id:namespace:mytype:g=group1:userspecific"; diff --git a/container-spifly/src/main/javadoc/README b/container-spifly/src/main/javadoc/README new file mode 100644 index 00000000000..6695538e308 --- /dev/null +++ b/container-spifly/src/main/javadoc/README @@ -0,0 +1 @@ +No javadoc available for module
\ No newline at end of file diff --git a/container-test/README.md b/container-test/README.md index 6c1de7859f8..529c9e49103 100644 --- a/container-test/README.md +++ b/container-test/README.md @@ -1,7 +1,13 @@ <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # Container-test -Convenience dependency for users developing OSGi bundles for JDisc. -Add this maven artifact as a **test** scope dependency in your pom.xml, -and it will transitively pull in all dependencies needed to unit test -JDisc components with the `application` test tool. +Convenience dependency for testing plugin components for the Jdisc container +with the `application` test tool. This artifact should contain all libraries +that are used internally by Vespa. Add this maven artifact as a **test** scope +dependency in your pom.xml to transitively pull in all dependencies needed to unit test +JDisc components. + +This should always be used in conjunction with a `provided` scoped dependency +on the `container` artifact, which contains all internal and 3rd party artifacts +that are exposed in Vespa's public APIs. (Internal Vespa developers should instead +use the `container-dev` artifact.) diff --git a/container-test/pom.xml b/container-test/pom.xml index a938329dd72..9e0ea06c801 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -26,8 +26,22 @@ <groupId>biz.aQute.bnd</groupId> <artifactId>*</artifactId> </exclusion> + <exclusion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + </exclusion> + <exclusion> + <groupId>org.bouncycastle</groupId> + <artifactId>bcpkix-jdk15on</artifactId> + </exclusion> </exclusions> </dependency> + <!-- Excluded above, and re-declared here, to work around a presumed Maven bug which otherwise puts this + in provided scope when container-test is declared before, and together with, the container artifact --> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcpkix-jdk15on</artifactId> + </dependency> <!-- All dependencies that should be visible in test classpath, but not compile classpath, for user projects must be added in compile scope here. These dependencies are explicitly excluded @@ -61,6 +75,11 @@ <groupId>com.google.code.findbugs</groupId> <artifactId>annotations</artifactId> </exclusion> + <exclusion> + <!-- We want the declaration from our parent, not the one in airlift --> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -98,14 +117,6 @@ <scope>compile</scope> </dependency> <dependency> - <groupId>org.bouncycastle</groupId> - <artifactId>bcpkix-jdk15on</artifactId> - </dependency> - <dependency> - <groupId>org.bouncycastle</groupId> - <artifactId>bcprov-jdk15on</artifactId> - </dependency> - <dependency> <!-- required for container-search code using org.json --> <groupId>org.json</groupId> <artifactId>json</artifactId> diff --git a/container/pom.xml b/container/pom.xml index b2a3a1baef8..4cda7b346b3 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -23,6 +23,10 @@ <version>${project.version}</version> <exclusions> <exclusion> + <groupId>org.bouncycastle</groupId> + <artifactId>bcpkix-jdk15on</artifactId> + </exclusion> + <exclusion> <groupId>org.ow2.asm</groupId> <artifactId>asm</artifactId> </exclusion> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ClusterId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ClusterId.java new file mode 100644 index 00000000000..0565a916dfa --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ClusterId.java @@ -0,0 +1,49 @@ +package com.yahoo.vespa.hosted.controller.api.identifiers; + +import com.yahoo.config.provision.ClusterSpec; + +import java.util.Objects; + +/** + * DeploymentId x ClusterSpec.Id = ClusterId + * + * @author ogronnesby + */ +public class ClusterId { + private final DeploymentId deploymentId; + private final ClusterSpec.Id clusterId; + + public ClusterId(DeploymentId deploymentId, ClusterSpec.Id clusterId) { + this.deploymentId = deploymentId; + this.clusterId = clusterId; + } + + public DeploymentId deploymentId() { + return deploymentId; + } + + public ClusterSpec.Id clusterId() { + return clusterId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClusterId clusterId1 = (ClusterId) o; + return Objects.equals(deploymentId, clusterId1.deploymentId) && Objects.equals(clusterId, clusterId1.clusterId); + } + + @Override + public int hashCode() { + return Objects.hash(deploymentId, clusterId); + } + + @Override + public String toString() { + return "ClusterId{" + + "deploymentId=" + deploymentId + + ", clusterId=" + clusterId + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index be83fd8de48..bf16913d05a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -28,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.secrets.GcpSecretStore; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretService; @@ -63,8 +62,6 @@ public interface ServiceRegistry { EndpointCertificateValidator endpointCertificateValidator(); - MeteringClient meteringService(); - ContactRetriever contactRetriever(); IssueHandler issueHandler(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java index 0efa225a437..7322c8e15f8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java @@ -65,12 +65,12 @@ public class ZtsClientMock implements ZtsClient { } @Override - public ZToken getRoleToken(AthenzDomain domain) { + public ZToken getRoleToken(AthenzDomain domain, Duration expiry) { throw new UnsupportedOperationException(); } @Override - public ZToken getRoleToken(AthenzRole athenzRole) { + public ZToken getRoleToken(AthenzRole athenzRole, Duration expiry) { throw new UnsupportedOperationException(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java index 6e60ec76199..b500cd1c133 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ClusterSpec; import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Objects; import java.util.Optional; /** @@ -131,6 +132,28 @@ public class Cluster { public Instant at() { return at; } public Optional<Instant> completion() { return completion; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ScalingEvent that = (ScalingEvent) o; + return Objects.equals(from, that.from) && Objects.equals(to, that.to) && Objects.equals(at, that.at) && Objects.equals(completion, that.completion); + } + + @Override + public int hashCode() { + return Objects.hash(from, to, at, completion); + } + + @Override + public String toString() { + return "ScalingEvent{" + + "from=" + from + + ", to=" + to + + ", at=" + at + + ", completion=" + completion + + '}'; + } } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 77879699ab9..09ddcf4fa5e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import java.util.Comparator; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; import static ai.vespa.validation.Validation.require; @@ -18,7 +18,6 @@ import static com.yahoo.config.provision.Environment.prod; import static com.yahoo.config.provision.Environment.staging; import static com.yahoo.config.provision.Environment.test; import static java.util.Comparator.naturalOrder; -import static java.util.stream.Collectors.toUnmodifiableList; /** * Specification for a deployment and/or test job to run: what zone, and whether it is a production test. @@ -27,6 +26,8 @@ import static java.util.stream.Collectors.toUnmodifiableList; */ public final class JobType implements Comparable<JobType> { + private static final RegionName unknown = RegionName.from("unknown"); + private final String jobName; private final ZoneId zone; private final boolean isProductionTest; @@ -38,18 +39,28 @@ public final class JobType implements Comparable<JobType> { } /** A system test in a test zone, or throws if no test zones are present.. */ - public static JobType systemTest(ZoneRegistry zones) { - return testIn(test, zones); + public static JobType systemTest(ZoneRegistry zones, CloudName cloud) { + return testIn(test, zones, cloud); } /** A staging test in a staging zone, or throws if no staging zones are present. */ - public static JobType stagingTest(ZoneRegistry zones){ - return testIn(staging, zones); + public static JobType stagingTest(ZoneRegistry zones, CloudName cloud){ + return testIn(staging, zones, cloud); } - private static JobType testIn(Environment environment, ZoneRegistry zones) { - return zones.zones().controllerUpgraded().in(environment).zones().stream().map(zone -> deploymentTo(zone.getId())) - .findFirst().orElseThrow(() -> new IllegalArgumentException("no zones in " + environment + " among " + zones.zones().controllerUpgraded().zones())); + /** Returns a test job in the given environment, preferring the given cloud, is possible; using the system cloud otherwise. */ + private static JobType testIn(Environment environment, ZoneRegistry zones, CloudName cloud) { + if (cloud == null) + return deploymentTo(ZoneId.from(environment, unknown)); + + ZoneList candidates = zones.zones().controllerUpgraded().in(environment); + if (candidates.in(cloud).zones().isEmpty()) + cloud = zones.systemZone().getCloudName(); + + return candidates.in(cloud).zones().stream().findFirst() + .map(zone -> deploymentTo(zone.getId())) + .orElseThrow(() -> new IllegalArgumentException("no zones in " + environment + " among " + + zones.zones().controllerUpgraded().zones())); } /** A deployment to the given dev region. */ @@ -118,27 +129,34 @@ public final class JobType implements Comparable<JobType> { throw new IllegalArgumentException("illegal serialized job type '" + raw + "'"); } - /** Creates a new job type from a job name, and a zone registry for looking up zones for the special system and staging test types. */ + /** + * Creates a new job type from a job name, and a zone registry for looking up zones for the special system and staging test types. + * Note: system and staging tests retrieved by job name always use the default cloud for the system! + */ public static JobType fromJobName(String jobName, ZoneRegistry zones) { - String[] parts = jobName.split("-", 2); - if (parts.length != 2) throw new IllegalArgumentException("job names must be 'system-test', 'staging-test', or environment and region parts, separated by '-', but got: " + jobName); - switch (parts[0]) { - case "system": return systemTest(zones); - case "staging": return stagingTest(zones); - case "production": return prod(parts[1]); - case "test": return test(parts[1]); - case "dev": return dev(parts[1]); - case "perf": return perf(parts[1]); - default: throw new IllegalArgumentException("job names must begin with one of: system, staging, production, test, dev, perf; but got: " + jobName); + switch (jobName) { + case "system-test": return systemTest(zones, null); + case "staging-test": return stagingTest(zones, null); } + String[] parts = jobName.split("-", 2); + if (parts.length == 2) + switch (parts[0]) { + case "production": return prod(parts[1]); + case "test": return test(parts[1]); + case "dev": return dev(parts[1]); + case "perf": return perf(parts[1]); + } + throw new IllegalArgumentException("job names must be 'system-test', 'staging-test', or <test|environment>-<region>, but got: " + jobName); } public static List<JobType> allIn(ZoneRegistry zones) { return zones.zones().reachable().zones().stream() .flatMap(zone -> zone.getEnvironment().isProduction() ? Stream.of(deploymentTo(zone.getId()), productionTestOf(zone.getId())) - : Stream.of(deploymentTo(zone.getId()))) + : zone.getEnvironment().isTest() ? Stream.of(deploymentTo(ZoneId.from(zone.getEnvironment(), unknown))) + : Stream.of(deploymentTo(zone.getId()))) + .distinct() .sorted(naturalOrder()) - .collect(toUnmodifiableList()); + .toList(); } /** A serialized form of this: {@code <environment>.<region>[.test]}; the inverse of {@link #ofSerialized(String)} */ @@ -152,6 +170,10 @@ public final class JobType implements Comparable<JobType> { /** Returns the zone for this job. */ public ZoneId zone() { + // sigh ... but the alternative is worse. + if (zone.region() == unknown) + throw new IllegalStateException("this job type was not initiated with a proper zone, programming error"); + return zone; } @@ -186,7 +208,7 @@ public final class JobType implements Comparable<JobType> { @Override public int compareTo(JobType other) { int result; - if (0 != (result = environment().compareTo(other.environment()))) return -result; + if (0 != (result = environment().compareTo(other.environment())) || environment().isTest()) return -result; if (0 != (result = zone.region().compareTo(other.zone.region()))) return result; return Boolean.compare(isProductionTest, other.isProductionTest); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java deleted file mode 100644 index 944a5eaf696..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.resource; - -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.TenantName; - -import java.time.YearMonth; -import java.util.Collection; -import java.util.List; - -/** - * Consumes and retrieves snapshots of resources allocated per application. - * - * @author olaa - */ -public interface MeteringClient { - - void consume(Collection<ResourceSnapshot> resources); - - List<ResourceSnapshot> getSnapshotHistoryForTenant(TenantName tenantName, YearMonth yearMonth); - - void refresh(); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java index 8ae12c0e7ac..49a24296fd3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java @@ -1,14 +1,21 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.resource; -import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; + +import java.time.Instant; import java.time.LocalDate; import java.time.YearMonth; import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -22,12 +29,18 @@ public interface ResourceDatabaseClient { void refreshMaterializedView(); + void writeScalingEvents(ClusterId clusterId, Collection<Cluster.ScalingEvent> scalingEvents); + + Map<ClusterId, List<Cluster.ScalingEvent>> scalingEvents(Instant from, Instant to, DeploymentId deploymentId); + Set<YearMonth> getMonthsWithSnapshotsForTenant(TenantName tenantName); List<ResourceSnapshot> getRawSnapshotHistoryForTenant(TenantName tenantName, YearMonth yearMonth); Set<TenantName> getTenants(); + Instant getOldestSnapshotTimestamp(Set<DeploymentId> deployments); + default List<ResourceUsage> getResourceSnapshotsForMonth(TenantName tenantName, YearMonth month) { return getResourceSnapshotsForPeriod(tenantName, getMonthStartTimeStamp(month), getMonthEndTimeStamp(month)); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java index c680990e240..f3abfd37b38 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java @@ -1,17 +1,19 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.resource; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; import java.math.BigDecimal; import java.time.Duration; import java.time.Instant; -import java.time.LocalDate; import java.time.YearMonth; -import java.time.ZoneId; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -33,6 +35,7 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { PlanRegistry planRegistry; Map<TenantName, Plan> planMap = new HashMap<>(); List<ResourceSnapshot> resourceSnapshots = new ArrayList<>(); + Map<ClusterId, List<Cluster.ScalingEvent>> scalingEvents = new HashMap<>(); private boolean hasRefreshedMaterializedView = false; public ResourceDatabaseClientMock(PlanRegistry planRegistry) { @@ -104,7 +107,7 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { public List<ResourceUsage> getResourceSnapshotsForPeriod(TenantName tenantName, long start, long end) { var tenantPlan = planMap.getOrDefault(tenantName, planRegistry.defaultPlan()); - var snapshotsPerDeployment = resourceSnapshots.stream() + return resourceSnapshots.stream() .filter(snapshot -> snapshot.getTimestamp().isAfter(Instant.ofEpochMilli(start))) .filter(snapshot -> snapshot.getTimestamp().isBefore(Instant.ofEpochMilli(end))) .filter(snapshot -> snapshot.getApplicationId().tenant().equals(tenantName)) @@ -117,8 +120,6 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toList()); - - return snapshotsPerDeployment; } @Override @@ -126,6 +127,21 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { hasRefreshedMaterializedView = true; } + @Override + public Instant getOldestSnapshotTimestamp(Set<DeploymentId> deployments) { + return Instant.ofEpochMilli(987654L); + } + + @Override + public void writeScalingEvents(ClusterId clusterId, Collection<Cluster.ScalingEvent> scalingEvents) { + this.scalingEvents.put(clusterId, List.copyOf(scalingEvents)); + } + + @Override + public Map<ClusterId, List<Cluster.ScalingEvent>> scalingEvents(Instant from, Instant to, DeploymentId deploymentId) { + return Map.of(); + } + public void setPlan(TenantName tenant, Plan plan) { planMap.put(tenant, plan); } @@ -133,4 +149,9 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { public boolean hasRefreshedMaterializedView() { return hasRefreshedMaterializedView; } + + public List<ResourceSnapshot> resourceSnapshots() { + return resourceSnapshots; + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java deleted file mode 100644 index ca094f98607..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.stubs; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; - -import java.time.YearMonth; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * @author olaa - */ -public class MockMeteringClient implements MeteringClient { - - private Collection<ResourceSnapshot> resources = new ArrayList<>(); - private Optional<MeteringData> meteringData; - private boolean isRefreshed = false; - - @Override - public void consume(Collection<ResourceSnapshot> resources){ - this.resources = resources; - } - - @Override - public List<ResourceSnapshot> getSnapshotHistoryForTenant(TenantName tenantName, YearMonth yearMonth) { - return new ArrayList<>(resources); - } - - @Override - public void refresh() { - isRefreshed = true; - } - - public Collection<ResourceSnapshot> consumedResources() { - return this.resources; - } - - public void setMeteringData(MeteringData meteringData) { - this.meteringData = Optional.of(meteringData); - this.resources = meteringData.getSnapshotHistory().entrySet().stream().map(Map.Entry::getValue).flatMap(List::stream).collect(Collectors.toList()); - } - - public boolean isRefreshed() { - return isRefreshed; - } -} diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java index 6ff52bd5f03..bbe0c2bd458 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java @@ -48,4 +48,4 @@ public class JobTypeTest { assertTrue(JobType.test("snohetta").isProduction()); } -} +}
\ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java index 8546eb5a971..281ac50e63a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java @@ -20,7 +20,6 @@ import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.time.Instant; @@ -88,15 +87,14 @@ public class ApplicationPackageValidator { /** Verify that each of the production zones listed in the deployment spec exist in this system */ private void validateSteps(DeploymentSpec deploymentSpec) { for (var spec : deploymentSpec.instances()) { - new DeploymentSteps(spec, controller.zoneRegistry()).jobs(); - spec.zones().stream() - .filter(zone -> zone.environment() == Environment.prod) - .forEach(zone -> { - if ( ! controller.zoneRegistry().hasZone(ZoneId.from(zone.environment(), - zone.region().orElseThrow()))) { - throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!"); - } - }); + for (var zone : spec.zones()) { + if (zone.environment().isManuallyDeployed()) + throw new IllegalArgumentException("region must be one with automated deployments, but got: " + zone.environment()); + + if ( zone.environment() == Environment.prod + && ! controller.zoneRegistry().hasZone(ZoneId.from(zone.environment(), zone.region().orElseThrow()))) + throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!"); + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index 33aeda5e011..0df70cd9c53 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -63,8 +63,10 @@ public class EndpointCertificates { if (duration.toSeconds() > 30) log.log(Level.INFO, Text.format("Getting endpoint certificate metadata for %s took %d seconds!", instance.id().serializedForm(), duration.toSeconds())); - if (controller.zoneRegistry().zones().ofCloud(CloudName.from("gcp")).ids().contains(zone)) { // Until CKMS is available from GCP + if (controller.zoneRegistry().zones().all().in(CloudName.from("gcp")).ids().contains(zone)) { // Until CKMS is available from GCP if(metadata.isPresent()) { + // Validate metadata before copying cert to GCP. This will ensure we don't bug out on the first deployment, but will take more time + certificateValidator.validate(metadata.get(), instance.id().serializedForm(), zone, controller.routing().certificateDnsNames(new DeploymentId(instance.id(), zone), deploymentSpec)); var m = metadata.get(); GcpSecretStore gcpSecretStore = controller.serviceRegistry().gcpSecretStore(); String mangledCertName = "endpointCert_" + m.certName().replace('.', '_') + "-v" + m.version(); // Google cloud does not accept dots in secrets, but they accept underscores diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index f0d172eeac6..24cd92d005f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableMap; -import com.yahoo.collections.Iterables; import com.yahoo.component.Version; import com.yahoo.component.VersionCompatibility; import com.yahoo.config.application.api.DeploymentInstanceSpec; @@ -11,9 +10,12 @@ import com.yahoo.config.application.api.DeploymentSpec.DeclaredTest; import com.yahoo.config.application.api.DeploymentSpec.DeclaredZone; import com.yahoo.config.application.api.DeploymentSpec.UpgradeRollout; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.stream.CustomCollectors; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; @@ -30,6 +32,7 @@ import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -54,8 +57,11 @@ import static java.util.Comparator.naturalOrder; import static java.util.Objects.requireNonNull; import static java.util.function.BinaryOperator.maxBy; import static java.util.stream.Collectors.collectingAndThen; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; import static java.util.stream.Collectors.toUnmodifiableList; /** @@ -71,11 +77,10 @@ public class DeploymentStatus { private final Application application; private final JobList allJobs; - private final JobType systemTest; - private final JobType stagingTest; private final VersionStatus versionStatus; private final Version systemVersion; private final Function<InstanceName, VersionCompatibility> versionCompatibility; + private final ZoneRegistry zones; private final Instant now; private final Map<JobId, StepStatus> jobSteps; private final List<StepStatus> allSteps; @@ -83,8 +88,7 @@ public class DeploymentStatus { public DeploymentStatus(Application application, Function<JobId, JobStatus> allJobs, ZoneRegistry zones, VersionStatus versionStatus, Version systemVersion, Function<InstanceName, VersionCompatibility> versionCompatibility, Instant now) { this.application = requireNonNull(application); - this.systemTest = JobType.systemTest(zones); - this.stagingTest = JobType.stagingTest(zones); + this.zones = zones; this.versionStatus = requireNonNull(versionStatus); this.systemVersion = requireNonNull(systemVersion); this.versionCompatibility = versionCompatibility; @@ -96,6 +100,14 @@ public class DeploymentStatus { this.allJobs = JobList.from(jobSteps.keySet().stream().map(allJobs).collect(toList())); } + private JobType systemTest(JobType dependent) { + return JobType.systemTest(zones, dependent == null ? null : findCloud(dependent)); + } + + private JobType stagingTest(JobType dependent) { + return JobType.stagingTest(zones, dependent == null ? null : findCloud(dependent)); + } + /** The application this deployment status concerns. */ public Application application() { return application; @@ -110,7 +122,7 @@ public class DeploymentStatus { private boolean hasFailures(StepStatus dependency, StepStatus dependent) { Set<StepStatus> dependents = new HashSet<>(); fillDependents(dependency, new HashSet<>(), dependents, dependent); - Set<JobId> criticalJobs = dependents.stream().flatMap(step -> step.job().stream()).collect(Collectors.toSet()); + Set<JobId> criticalJobs = dependents.stream().flatMap(step -> step.job().stream()).collect(toSet()); return ! allJobs.matching(job -> criticalJobs.contains(job.id())) .failingHard() @@ -148,8 +160,7 @@ public class DeploymentStatus { public Map<JobType, JobStatus> instanceJobs(InstanceName instance) { return allJobs.asList().stream() .filter(job -> job.id().application().equals(application.id().instance(instance))) - .collect(Collectors.toUnmodifiableMap(job -> job.id().type(), - Function.identity())); + .collect(CustomCollectors.toLinkedMap(job -> job.id().type(), Function.identity())); } /** Filterable job status lists for each instance of this application. */ @@ -204,16 +215,26 @@ public class DeploymentStatus { if (change == null || ! change.hasTargets()) return; - Optional<JobId> firstProductionJobWithDeployment = jobSteps.keySet().stream() - .filter(jobId -> jobId.type().isProduction() && jobId.type().isDeployment()) - .filter(jobId -> deploymentFor(jobId).isPresent()) - .findFirst(); - Versions versions = Versions.from(change, - application, - firstProductionJobWithDeployment.flatMap(this::deploymentFor), - fallbackPlatform(change, job)); - if (step.completedAt(change, Optional.empty()).isEmpty()) - jobs.merge(job, List.of(new Job(job.type(), versions, step.readyAt(change), change)), DeploymentStatus::union); + Collection<Optional<JobId>> firstProductionJobsWithDeployment = jobSteps.keySet().stream() + .filter(jobId -> jobId.type().isProduction() && jobId.type().isDeployment()) + .filter(jobId -> deploymentFor(jobId).isPresent()) + .collect(groupingBy(jobId -> findCloud(jobId.type()), + Collectors.reducing((o, n) -> o))) // Take the first. + .values(); + if (firstProductionJobsWithDeployment.isEmpty()) + firstProductionJobsWithDeployment = List.of(Optional.empty()); + + for (Optional<JobId> firstProductionJobWithDeploymentInCloud : firstProductionJobsWithDeployment) { + Versions versions = Versions.from(change, + application, + firstProductionJobWithDeploymentInCloud.flatMap(this::deploymentFor), + fallbackPlatform(change, job)); + if (step.completedAt(change, firstProductionJobWithDeploymentInCloud).isEmpty()) { + CloudName cloud = firstProductionJobWithDeploymentInCloud.map(JobId::type).map(this::findCloud).orElse(zones.systemZone().getCloudName()); + JobType typeWithZone = job.type().isSystemTest() ? JobType.systemTest(zones, cloud) : JobType.stagingTest(zones, cloud); + jobs.merge(job, List.of(new Job(typeWithZone, versions, step.readyAt(change), change)), DeploymentStatus::union); + } + } }); return Collections.unmodifiableMap(jobs); } @@ -260,14 +281,7 @@ public class DeploymentStatus { /** The step status for all relevant steps in the deployment spec of this, in the same order as in the deployment spec. */ public List<StepStatus> allSteps() { - if (allSteps.isEmpty()) - return List.of(); - - List<JobId> firstTestJobs = List.of(firstDeclaredOrElseImplicitTest(systemTest), - firstDeclaredOrElseImplicitTest(stagingTest)); - return allSteps.stream() - .filter(step -> step.isDeclared() || firstTestJobs.contains(step.job().orElseThrow())) - .collect(toUnmodifiableList()); + return allSteps; } public Optional<Deployment> deploymentFor(JobId job) { @@ -276,13 +290,30 @@ public class DeploymentStatus { } private <T extends Comparable<T>> Optional<T> newestTested(InstanceName instance, Function<Run, T> runMapper) { - return instanceJobs().get(application.id().instance(instance)) - .type(systemTest, stagingTest) - .asList().stream().flatMap(jobs -> jobs.runs().values().stream()) - .filter(Run::hasSucceeded) - .map(runMapper) - .max(naturalOrder()); + Set<CloudName> clouds = Stream.concat(Stream.of(zones.systemZone().getCloudName()), + jobSteps.keySet().stream() + .filter(job -> job.type().isProduction()) + .map(job -> findCloud(job.type()))) + .collect(toSet()); + List<ZoneId> testZones = new ArrayList<>(); + if (application.deploymentSpec().requireInstance(instance).concerns(test)) + for (CloudName cloud: clouds) testZones.add(JobType.systemTest(zones, cloud).zone()); + if (application.deploymentSpec().requireInstance(instance).concerns(staging)) + for (CloudName cloud: clouds) testZones.add(JobType.stagingTest(zones, cloud).zone()); + + Map<ZoneId, Optional<T>> newestPerZone = instanceJobs().get(application.id().instance(instance)) + .type(systemTest(null), stagingTest(null)) + .asList().stream().flatMap(jobs -> jobs.runs().values().stream()) + .filter(Run::hasSucceeded) + .collect(groupingBy(run -> run.id().type().zone(), + mapping(runMapper, Collectors.maxBy(naturalOrder())))); + return newestPerZone.keySet().containsAll(testZones) + ? testZones.stream().map(newestPerZone::get) + .reduce((o, n) -> o.isEmpty() || n.isEmpty() ? Optional.empty() : n.get().compareTo(o.get()) < 0 ? n : o) + .orElse(Optional.empty()) + : Optional.empty(); } + /** * The change to a revision which all dependencies of the given instance has completed, * which does not downgrade any deployments in the instance, @@ -347,8 +378,8 @@ public class DeploymentStatus { .filter(run -> run.versions().equals(versions)) .findFirst()) .map(Run::start); - Optional<Instant> systemTestedAt = testedAt(job.application(), systemTest, versions); - Optional<Instant> stagingTestedAt = testedAt(job.application(), stagingTest, versions); + Optional<Instant> systemTestedAt = testedAt(job.application(), systemTest(job.type()), versions); + Optional<Instant> stagingTestedAt = testedAt(job.application(), stagingTest(job.type()), versions); if (systemTestedAt.isEmpty() || stagingTestedAt.isEmpty()) return triggeredAt; Optional<Instant> testedAt = systemTestedAt.get().isAfter(stagingTestedAt.get()) ? systemTestedAt : stagingTestedAt; return triggeredAt.isPresent() && triggeredAt.get().isBefore(testedAt.get()) ? triggeredAt : testedAt; @@ -357,14 +388,15 @@ public class DeploymentStatus { /** Earliest instant when versions were tested for the given instance */ private Optional<Instant> testedAt(ApplicationId instance, JobType type, Versions versions) { return declaredTest(instance, type).map(__ -> allJobs.instance(instance.instance())) - .orElse(allJobs) - .type(type).asList().stream() - .flatMap(status -> RunList.from(status) - .on(versions) - .matching(Run::hasSucceeded) - .asList().stream() - .map(Run::start)) - .min(naturalOrder()); + .orElse(allJobs) + .type(type).asList().stream() + .flatMap(status -> RunList.from(status) + .on(versions) + .matching(run -> run.id().type().zone().equals(type.zone())) + .matching(Run::hasSucceeded) + .asList().stream() + .map(Run::start)) + .min(naturalOrder()); } private Map<JobId, List<Job>> productionJobs(InstanceName instance, Change change, boolean assumeUpgradesSucceed) { @@ -484,7 +516,7 @@ public class DeploymentStatus { // Both changes are ready for this step, and we look to the specified rollout to decide. boolean platformReadyFirst = platformReadyAt.get().isBefore(revisionReadyAt.get()); boolean revisionReadyFirst = revisionReadyAt.get().isBefore(platformReadyAt.get()); - boolean failingUpgradeOnlyTests = ! jobs().type(systemTest, stagingTest) + boolean failingUpgradeOnlyTests = ! jobs().type(systemTest(job.type()), stagingTest(job.type())) .failingHardOn(Versions.from(change.withoutApplication(), application, deploymentFor(job), systemVersion)) .isEmpty(); switch (rollout) { @@ -507,12 +539,12 @@ public class DeploymentStatus { /** The test jobs that need to run prior to the given production deployment jobs. */ public Map<JobId, List<Job>> testJobs(Map<JobId, List<Job>> jobs) { Map<JobId, List<Job>> testJobs = new LinkedHashMap<>(); - for (JobType testType : List.of(systemTest, stagingTest)) { - jobs.forEach((job, versionsList) -> { + jobs.forEach((job, versionsList) -> { + for (JobType testType : List.of(systemTest(job.type()), stagingTest(job.type()))) { if (job.type().isProduction() && job.type().isDeployment()) { declaredTest(job.application(), testType).ifPresent(testJob -> { for (Job productionJob : versionsList) - if (allJobs.successOn(productionJob.versions()).get(testJob).isEmpty()) + if (allJobs.successOn(testType, productionJob.versions()).asList().isEmpty()) testJobs.merge(testJob, List.of(new Job(testJob.type(), productionJob.versions(), jobSteps().get(testJob).readyAt(productionJob.change), @@ -520,13 +552,15 @@ public class DeploymentStatus { DeploymentStatus::union); }); } - }); - jobs.forEach((job, versionsList) -> { + } + }); + jobs.forEach((job, versionsList) -> { + for (JobType testType : List.of(systemTest(job.type()), stagingTest(job.type()))) { for (Job productionJob : versionsList) if ( job.type().isProduction() && job.type().isDeployment() - && allJobs.successOn(productionJob.versions()).type(testType).isEmpty() + && allJobs.successOn(testType, productionJob.versions()).asList().isEmpty() && testJobs.keySet().stream() - .noneMatch(test -> test.type().equals(testType) + .noneMatch(test -> test.type().equals(testType) && test.type().zone().equals(testType.zone()) && testJobs.get(test).stream().anyMatch(testJob -> testJob.versions().equals(productionJob.versions())))) { JobId testJob = firstDeclaredOrElseImplicitTest(testType); testJobs.merge(testJob, @@ -536,11 +570,15 @@ public class DeploymentStatus { productionJob.change)), DeploymentStatus::union); } - }); - } + } + }); return Collections.unmodifiableMap(testJobs); } + private CloudName findCloud(JobType job) { + return zones.zones().all().get(job.zone()).map(ZoneApi::getCloudName).orElse(zones.systemZone().getCloudName()); + } + private JobId firstDeclaredOrElseImplicitTest(JobType testJob) { return application.deploymentSpec().instanceNames().stream() .map(name -> new JobId(application.id().instance(name), testJob)) @@ -596,7 +634,7 @@ public class DeploymentStatus { JobId jobId; StepStatus stepStatus; if (step.concerns(test) || step.concerns(staging)) { - jobType = step.concerns(test) ? systemTest : stagingTest; + jobType = step.concerns(test) ? systemTest(null) : stagingTest(null); jobId = new JobId(application.id().instance(instance), jobType); stepStatus = JobStepStatus.ofTestDeployment((DeclaredZone) step, List.of(), this, jobs.apply(jobId), true); previous = new ArrayList<>(previous); @@ -622,19 +660,19 @@ public class DeploymentStatus { if (step instanceof DeploymentInstanceSpec) { DeploymentInstanceSpec spec = ((DeploymentInstanceSpec) step); - StepStatus instanceStatus = new InstanceStatus(spec, previous, now, application.require(spec.name())); + StepStatus instanceStatus = new InstanceStatus(spec, previous, now, application.require(spec.name()), this); instance = spec.name(); allSteps.add(instanceStatus); previous = List.of(instanceStatus); if (instance.equals(implicitSystemTest)) { - JobId job = new JobId(application.id().instance(instance), systemTest); + JobId job = new JobId(application.id().instance(instance), systemTest(null)); JobStepStatus testStatus = JobStepStatus.ofTestDeployment(new DeclaredZone(test), List.of(), this, jobs.apply(job), false); dependencies.put(job, testStatus); allSteps.add(testStatus); } if (instance.equals(implicitStagingTest)) { - JobId job = new JobId(application.id().instance(instance), stagingTest); + JobId job = new JobId(application.id().instance(instance), stagingTest(null)); JobStepStatus testStatus = JobStepStatus.ofTestDeployment(new DeclaredZone(staging), List.of(), this, jobs.apply(job), false); dependencies.put(job, testStatus); @@ -775,13 +813,25 @@ public class DeploymentStatus { private final DeploymentInstanceSpec spec; private final Instant now; private final Instance instance; + private final DeploymentStatus status; private InstanceStatus(DeploymentInstanceSpec spec, List<StepStatus> dependencies, Instant now, - Instance instance) { + Instance instance, DeploymentStatus status) { super(StepType.instance, spec, dependencies, spec.name()); this.spec = spec; this.now = now; this.instance = instance; + this.status = status; + } + + /** The time at which this step is ready to run the specified change and / or versions. */ + @Override + public Optional<Instant> readyAt(Change change) { + return status.jobSteps.keySet().stream() + .filter(job -> job.type().isProduction() && job.application().instance().equals(instance.name())) + .map(job -> super.readyAt(change, Optional.of(job))) + .reduce((o, n) -> o.isEmpty() || n.isEmpty() ? Optional.empty() : n.get().isBefore(o.get()) ? n : o) + .orElseGet(() -> super.readyAt(change, Optional.empty())); } /** @@ -936,6 +986,8 @@ public class DeploymentStatus { return new JobStepStatus(StepType.test, step, dependencies, job, status) { @Override Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { + Optional<ZoneId> requiredTestZone = dependent.map(dep -> job.id().type().isSystemTest() ? status.systemTest(dep.type()).zone() + : status.stagingTest(dep.type()).zone()); return RunList.from(job) .matching(run -> dependent.flatMap(status::deploymentFor) .map(deployment -> run.versions().targetsMatch(Versions.from(change, @@ -945,6 +997,7 @@ public class DeploymentStatus { .orElseGet(() -> (change.platform().isEmpty() || change.platform().get().equals(run.versions().targetPlatform())) && (change.revision().isEmpty() || change.revision().get().equals(run.versions().targetRevision())))) .matching(Run::hasSucceeded) + .matching(run -> requiredTestZone.isEmpty() || requiredTestZone.get().equals(run.id().type().zone())) .asList().stream() .map(run -> run.end().get()) .max(naturalOrder()); @@ -959,16 +1012,22 @@ public class DeploymentStatus { public static class Job { + private final JobType type; private final Versions versions; private final Optional<Instant> readyAt; private final Change change; public Job(JobType type, Versions versions, Optional<Instant> readyAt, Change change) { + this.type = type; this.versions = type.isSystemTest() ? versions.withoutSources() : versions; this.readyAt = readyAt; this.change = change; } + public JobType type() { + return type; + } + public Versions versions() { return versions; } @@ -982,12 +1041,12 @@ public class DeploymentStatus { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Job job = (Job) o; - return versions.equals(job.versions) && readyAt.equals(job.readyAt) && change.equals(job.change); + return type.zone().equals(job.type.zone()) && versions.equals(job.versions) && readyAt.equals(job.readyAt) && change.equals(job.change); } @Override public int hashCode() { - return Objects.hash(versions, readyAt, change); + return Objects.hash(type.zone(), versions, readyAt, change); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java deleted file mode 100644 index 44079a90097..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.application.Deployment; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Comparator.comparingInt; -import static java.util.stream.Collectors.collectingAndThen; - -/** - * This class provides helper methods for reading a deployment spec. - * - * @author mpolden - */ -public class DeploymentSteps { - - private final DeploymentInstanceSpec spec; - private final ZoneRegistry zones; - - public DeploymentSteps(DeploymentInstanceSpec spec, ZoneRegistry zones) { - this.spec = Objects.requireNonNull(spec, "spec cannot be null"); - this.zones = Objects.requireNonNull(zones, "system cannot be null"); - } - - /** Returns jobs for this, in the order they should run */ - public List<JobType> jobs() { - return Stream.concat(production().isEmpty() ? Stream.of() : Stream.of(JobType.systemTest(zones), JobType.stagingTest(zones)), - spec.steps().stream().flatMap(step -> toJobs(step).stream())) - .distinct() - .collect(Collectors.toUnmodifiableList()); - } - - /** Returns job status sorted according to deployment spec */ - public List<JobStatus> sortedJobs(Collection<JobStatus> jobStatus) { - List<JobType> sortedJobs = jobs(); - return jobStatus.stream() - .sorted(comparingInt(job -> sortedJobs.indexOf(job.id().type()))) - .collect(Collectors.toUnmodifiableList()); - } - - /** Returns deployments sorted according to declared zones */ - public List<Deployment> sortedDeployments(Collection<Deployment> deployments) { - List<ZoneId> productionZones = spec.zones().stream() - .filter(z -> z.region().isPresent()) - .map(z -> ZoneId.from(z.environment(), z.region().get())) - .collect(Collectors.toUnmodifiableList()); - return deployments.stream() - .sorted(comparingInt(deployment -> productionZones.indexOf(deployment.zone()))) - .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); - } - - /** Resolve jobs from step */ - public List<JobType> toJobs(DeploymentSpec.Step step) { - return step.zones().stream() - .map(this::toJob) - .collect(Collectors.toUnmodifiableList()); - } - - /** Returns test jobs to run for this spec */ - public List<JobType> testJobs() { - return jobs().stream().filter(type -> type.environment().isTest()).collect(Collectors.toUnmodifiableList()); - } - - /** Returns declared production jobs in this */ - public List<JobType> productionJobs() { - return toJobs(production()); - } - - /** Returns declared production steps in this */ - public List<DeploymentSpec.Step> production() { - return spec.steps().stream() - .filter(step -> ! isTest(step)) - .collect(Collectors.toUnmodifiableList()); - } - - private boolean isTest(DeploymentSpec.Step step) { - return step.concerns(Environment.test) || step.concerns(Environment.staging); - } - - /** Resolve job from deployment zone */ - private JobType toJob(DeploymentSpec.DeclaredZone zone) { - switch (zone.environment()) { - case prod: return JobType.prod(zone.region().get()); - case test: return JobType.systemTest(zones); - case staging: return JobType.stagingTest(zones); - default: throw new IllegalArgumentException("region must be one with automated deployments, but got: " + zone.environment()); - } - } - - /** Resolve jobs from steps */ - private List<JobType> toJobs(List<DeploymentSpec.Step> steps) { - return steps.stream() - .flatMap(step -> toJobs(step).stream()) - .collect(Collectors.toUnmodifiableList()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 419d6155fe7..c28f94bc4d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; @@ -24,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import java.math.BigDecimal; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -71,10 +73,6 @@ public class DeploymentTrigger { this.jobs = controller.jobController(); } - public DeploymentSteps steps(DeploymentInstanceSpec spec) { - return new DeploymentSteps(spec, controller.zoneRegistry()); - } - /** * Propagates the latest revision to ready instances. * Ready instances are those whose dependencies are complete, and which aren't blocked, and, additionally, @@ -108,8 +106,9 @@ public class DeploymentTrigger { // If the outstanding revision requires a certain platform for compatibility, add that here. VersionCompatibility compatibility = applications().versionCompatibility(status.application().id().instance(instance)); Predicate<Version> compatibleWithCompileVersion = version -> compileVersion.map(compiled -> compatibility.accept(version, compiled)).orElse(true); - if (status.application().productionDeployments().getOrDefault(instance, List.of()).stream() - .anyMatch(deployment -> ! compatibleWithCompileVersion.test(deployment.version()))) { + if ( status.application().productionDeployments().isEmpty() + || status.application().productionDeployments().getOrDefault(instance, List.of()).stream() + .anyMatch(deployment -> ! compatibleWithCompileVersion.test(deployment.version()))) { return targetsForPolicy(controller.readVersionStatus(), status.application().deploymentSpec().requireInstance(instance).upgradePolicy()) .stream() // Pick the latest platform which is compatible with the compile version, and is ready for this instance. .filter(compatibleWithCompileVersion) @@ -369,6 +368,7 @@ public class DeploymentTrigger { .withDeploymentSpec()) .withChanges() .asList().stream() + .filter(status -> ! hasExceededQuota(status.application().id().tenant())) .map(this::computeReadyJobs) .flatMap(Collection::stream) .collect(toList()); @@ -378,21 +378,25 @@ public class DeploymentTrigger { private List<Job> computeReadyJobs(DeploymentStatus status) { List<Job> jobs = new ArrayList<>(); Map<JobId, List<DeploymentStatus.Job>> jobsToRun = status.jobsToRun(); - jobsToRun.forEach((job, versionsList) -> { - versionsList.get(0).readyAt() - .filter(readyAt -> ! clock.instant().isBefore(readyAt)) - .filter(__ -> ! (job.type().isProduction() && isUnhealthyInAnotherZone(status.application(), job))) - .filter(__ -> abortIfRunning(status, jobsToRun, job)) // Abort and trigger this later if running with outdated parameters. - .map(readyAt -> deploymentJob(status.application().require(job.application().instance()), - versionsList.get(0).versions(), - job.type(), - status.instanceJobs(job.application().instance()).get(job.type()), - readyAt)) - .ifPresent(jobs::add); + jobsToRun.forEach((jobId, jobsList) -> { + DeploymentStatus.Job job = jobsList.get(0); + if ( job.readyAt().isPresent() + && ! clock.instant().isBefore(job.readyAt().get()) + && ! (jobId.type().isProduction() && isUnhealthyInAnotherZone(status.application(), jobId)) + && abortIfRunning(status, jobsToRun, jobId)) // Abort and trigger this later if running with outdated parameters. + jobs.add(deploymentJob(status.application().require(jobId.application().instance()), + job.versions(), + job.type(), + status.instanceJobs(jobId.application().instance()).get(jobId.type()), + job.readyAt().get())); }); return Collections.unmodifiableList(jobs); } + private boolean hasExceededQuota(TenantName tenant) { + return controller.serviceRegistry().billingController().getQuota(tenant).budget().equals(Optional.of(BigDecimal.ZERO)); + } + /** Returns whether the application is healthy in all other production zones. */ private boolean isUnhealthyInAnotherZone(Application application, JobId job) { for (Deployment deployment : application.require(job.application().instance()).productionDeployments().values()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 35eaa36cd2f..5113d386b23 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableSortedMap; import com.yahoo.component.Version; import com.yahoo.component.VersionCompatibility; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; @@ -41,6 +42,7 @@ import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Deque; @@ -627,7 +629,7 @@ public class JobController { DeploymentId deploymentId = new DeploymentId(id, type.zone()); Optional<Run> lastRun = last(id, type); - lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); + lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id(), Duration.ofMinutes(2))); long build = 1 + lastRun.map(run -> run.versions().targetRevision().number()).orElse(0L); RevisionId revisionId = RevisionId.forDevelopment(build, new JobId(id, type)); @@ -668,14 +670,6 @@ public class JobController { } private Version findTargetPlatform(ApplicationPackage applicationPackage, DeploymentId id, Optional<Instance> instance) { - Optional<Integer> major = applicationPackage.deploymentSpec().majorVersion(); - if (major.isPresent()) - return controller.applications().lastCompatibleVersion(major.get()) - .orElseThrow(() -> new IllegalArgumentException("major " + major.get() + " specified in deployment.xml, " + - "but no version on this major was found")); - - VersionCompatibility compatibility = controller.applications().versionCompatibility(id.applicationId()); - // Prefer previous platform if possible. Candidates are all deployable, ascending, with existing version appended; then reversed. List<Version> versions = controller.readVersionStatus().deployableVersions().stream() .map(VespaVersion::versionNumber) @@ -685,24 +679,47 @@ public class JobController { .map(Deployment::version) .ifPresent(versions::add); + if (versions.isEmpty()) + throw new IllegalStateException("no deployable platform version found in the system"); + + VersionCompatibility compatibility = controller.applications().versionCompatibility(id.applicationId()); + List<Version> compatibleVersions = new ArrayList<>(); for (Version target : reversed(versions)) if (applicationPackage.compileVersion().isEmpty() || compatibility.accept(target, applicationPackage.compileVersion().get())) + compatibleVersions.add(target); + + if (compatibleVersions.isEmpty()) + throw new IllegalArgumentException("no platforms are compatible with compile version " + applicationPackage.compileVersion().get()); + + Optional<Integer> major = applicationPackage.deploymentSpec().majorVersion(); + List<Version> versionOnRightMajor = new ArrayList<>(); + for (Version target : reversed(versions)) + if (major.isEmpty() || major.get() == target.getMajor()) + versionOnRightMajor.add(target); + + if (versionOnRightMajor.isEmpty()) + throw new IllegalArgumentException("no platforms were found for major version " + major.get() + " specified in deployment.xml"); + + for (Version target : compatibleVersions) + if (versionOnRightMajor.contains(target)) return target; - throw new IllegalArgumentException("no suitable platform version found" + - applicationPackage.compileVersion() - .map(version -> " for package compiled against " + version) - .orElse("")); + throw new IllegalArgumentException("no platforms on major version " + major.get() + " specified in deployment.xml " + + "are compatible with compile version " + applicationPackage.compileVersion().get()); } /** Aborts a run and waits for it complete. */ - private void abortAndWait(RunId id) { + private void abortAndWait(RunId id, Duration timeout) { abort(id, "replaced by new deployment"); runner.get().accept(last(id.application(), id.type()).get()); + Instant doom = controller.clock().instant().plus(timeout); + Duration sleep = Duration.ofMillis(100); while ( ! last(id.application(), id.type()).get().hasEnded()) { + if (controller.clock().instant().plus(sleep).isAfter(doom)) + throw new UncheckedTimeoutException("timeout waiting for " + id + " to abort and finish"); try { - Thread.sleep(100); + Thread.sleep(sleep.toMillis()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java index 387ea755414..551f841233e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java @@ -119,8 +119,12 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { } /** Returns the jobs with successful runs matching the given versions — targets only for system test, everything present otherwise. */ - public JobList successOn(Versions versions) { - return matching(job -> ! RunList.from(job).matching(Run::hasSucceeded).on(versions).isEmpty()); + public JobList successOn(JobType type, Versions versions) { + return matching(job -> job.id().type().equals(type) + && ! RunList.from(job) + .matching(run -> run.hasSucceeded() && run.id().type().zone().equals(type.zone())) + .on(versions) + .isEmpty()); } // ----------------------------------- JobRun filtering diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index f7368aab143..4aeecdcd4ff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -58,7 +58,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer)); maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher)); maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer())); - maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().meteringService())); + maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().resourceDatabase())); maintainers.add(new ResourceTagMaintainer(controller, intervals.resourceTagMaintainer, controller.serviceRegistry().resourceTagger())); maintainers.add(new ApplicationMetaDataGarbageCollector(controller, intervals.applicationMetaDataGarbageCollector)); maintainers.add(new ArtifactExpirer(controller, intervals.containerImageExpirer)); @@ -75,6 +75,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new RetriggerMaintainer(controller, intervals.retriggerMaintainer)); maintainers.add(new UserManagementMaintainer(controller, intervals.userManagementMaintainer, controller.serviceRegistry().roleMaintainer())); maintainers.add(new BillingDatabaseMaintainer(controller, intervals.billingDatabaseMaintainer)); + maintainers.add(new MeteringMonitorMaintainer(controller, intervals.meteringMonitorMaintainer, controller.serviceRegistry().resourceDatabase(), metric)); } public Upgrader upgrader() { return upgrader; } @@ -131,6 +132,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration retriggerMaintainer; private final Duration userManagementMaintainer; private final Duration billingDatabaseMaintainer; + private final Duration meteringMonitorMaintainer; public Intervals(SystemName system) { this.system = Objects.requireNonNull(system); @@ -164,6 +166,7 @@ public class ControllerMaintenance extends AbstractComponent { this.retriggerMaintainer = duration(1, MINUTES); this.userManagementMaintainer = duration(12, HOURS); this.billingDatabaseMaintainer = duration(5, MINUTES); + this.meteringMonitorMaintainer = duration(30, MINUTES); } private Duration duration(long amount, TemporalUnit unit) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java new file mode 100644 index 00000000000..eadbdf74c3c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainer.java @@ -0,0 +1,64 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; + +import java.time.Duration; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Reports discrepancies between currently deployed applications and + * recently stored metering data in ResourceDatabaseClient. + * + * @author olaa + */ +public class MeteringMonitorMaintainer extends ControllerMaintainer { + + private final ResourceDatabaseClient resourceDatabaseClient; + private final Metric metric; + + protected static final String METERING_AGE_METRIC_NAME = "metering.age.seconds"; + private static final Logger logger = Logger.getLogger(MeteringMonitorMaintainer.class.getName()); + + public MeteringMonitorMaintainer(Controller controller, Duration interval, ResourceDatabaseClient resourceDatabaseClient, Metric metric) { + super(controller, interval, null, SystemName.allOf(SystemName::isPublic)); + this.resourceDatabaseClient = resourceDatabaseClient; + this.metric = metric; + } + + @Override + protected double maintain() { + var activeDeployments = activeDeployments(); + var lastSnapshotTime = resourceDatabaseClient.getOldestSnapshotTimestamp(activeDeployments); + var age = controller().clock().instant().getEpochSecond() - lastSnapshotTime.getEpochSecond(); + metric.set(METERING_AGE_METRIC_NAME, age, metric.createContext(Collections.emptyMap())); + return 1; + } + + private Set<DeploymentId> activeDeployments() { + return controller().applications().asList() + .stream() + .flatMap(app -> app.instances().values().stream()) + .flatMap(this::instancesToDeployments) + .collect(Collectors.toSet()); + } + + private Stream<DeploymentId> instancesToDeployments(Instance instance) { + return instance.deployments() + .keySet() + .stream() + .map(deployment -> new DeploymentId(instance.id(), deployment)); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index 51b40a9a4c7..3bd1c7bb358 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -65,7 +65,7 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } private Release releaseIn(CloudName cloud) { - boolean useTaggedRelease = controller().zoneRegistry().zones().reprovisionToUpgradeOs().ofCloud(cloud) + boolean useTaggedRelease = controller().zoneRegistry().zones().all().reprovisionToUpgradeOs().in(cloud) .zones().isEmpty(); if (useTaggedRelease) { return new TaggedRelease(controller().system(), controller().serviceRegistry().artifactRepository()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java index 5178918aa48..974345330aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java @@ -20,7 +20,8 @@ public class ReadyJobsTrigger extends ControllerMaintainer { @Override public double maintain() { TriggerResult result = controller().applications().deploymentTrigger().triggerReadyJobs(); - return result.triggered() * 1.0f / (result.triggered() + result.failed()); + long total = result.triggered() + result.failed(); + return total == 0 ? 1 : (double) result.triggered() / (result.triggered() + result.failed()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index d4905f7e20a..892ad669e4b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -10,13 +10,18 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -37,6 +42,7 @@ import java.util.function.Function; import java.util.logging.Level; import java.util.stream.Collector; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Creates a {@link ResourceSnapshot} per application, which is then passed on to a MeteringClient @@ -57,7 +63,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { private final ApplicationController applications; private final NodeRepository nodeRepository; - private final MeteringClient meteringClient; + private final ResourceDatabaseClient resourceClient; private final CuratorDb curator; private final SystemName systemName; private final Metric metric; @@ -71,11 +77,11 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { public ResourceMeterMaintainer(Controller controller, Duration interval, Metric metric, - MeteringClient meteringClient) { + ResourceDatabaseClient resourceClient) { super(controller, interval); this.applications = controller.applications(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.meteringClient = meteringClient; + this.resourceClient = resourceClient; this.curator = controller.curator(); this.systemName = controller.serviceRegistry().zoneRegistry().system(); this.metric = metric; @@ -94,6 +100,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { } if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots); + if (systemName.isPublic() && systemName.isCd()) reportAllScalingEvents(); updateDeploymentCost(resourceSnapshots); return 1.0; } @@ -124,13 +131,13 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { } private void reportResourceSnapshots(Collection<ResourceSnapshot> resourceSnapshots) { - meteringClient.consume(resourceSnapshots); + resourceClient.writeResourceSnapshots(resourceSnapshots); updateMeteringMetrics(resourceSnapshots); try (var lock = curator.lockMeteringRefreshTime()) { if (needsRefresh(curator.readMeteringRefreshTime())) { - meteringClient.refresh(); + resourceClient.refreshMaterializedView(); curator.writeMeteringRefreshTime(clock.millis()); } } catch (TimeoutException ignored) { @@ -147,6 +154,37 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { .collect(Collectors.toList()); } + private Stream<Instance> mapApplicationToInstances(Application application) { + return application.instances().values().stream(); + } + + private Stream<DeploymentId> mapInstanceToDeployments(Instance instance) { + return instance.deployments().keySet().stream().map(zoneId -> { + return new DeploymentId(instance.id(), zoneId); + }); + } + + private Stream<Map.Entry<ClusterId, List<Cluster.ScalingEvent>>> mapDeploymentToClusterScalingEvent(DeploymentId deploymentId) { + return nodeRepository.getApplication(deploymentId.zoneId(), deploymentId.applicationId()) + .clusters().entrySet().stream() + .map(cluster -> Map.entry(new ClusterId(deploymentId, cluster.getKey()), cluster.getValue().scalingEvents())); + } + + private void reportAllScalingEvents() { + var clusters = controller().applications().asList().stream() + .flatMap(this::mapApplicationToInstances) + .flatMap(this::mapInstanceToDeployments) + .flatMap(this::mapDeploymentToClusterScalingEvent) + .collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue + )); + + for (var cluster : clusters.entrySet()) { + resourceClient.writeScalingEvents(cluster.getKey(), cluster.getValue()); + } + } + private Collection<ResourceSnapshot> createResourceSnapshotsFromNodes(ZoneId zoneId, List<Node> nodes) { return nodes.stream() .filter(this::unlessNodeOwnerIsSystemApplication) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java index 7b9c24df0fa..3588ae53a74 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -34,8 +34,8 @@ public class ResourceTagMaintainer extends ControllerMaintainer { @Override public double maintain() { controller().zoneRegistry().zones() - .ofCloud(CloudName.from("aws")) .reachable() + .in(CloudName.from("aws")) .zones().forEach(zone -> { Map<HostName, ApplicationId> applicationOfHosts = getTenantOfParentHosts(zone.getId()); int taggedResources = resourceTagger.tagResources(zone, applicationOfHosts); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java index b36b2b9cad8..31f79c78ad5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java @@ -48,7 +48,7 @@ public class CostCalculator { // Sum up allocations Map<Property, ResourceAllocation> allocationByProperty = new HashMap<>(); var nodes = controller.zoneRegistry().zones() - .reachable().in(Environment.prod).ofCloud(cloudName).zones().stream() + .reachable().in(Environment.prod).in(cloudName).zones().stream() .flatMap(zone -> uncheck(() -> nodeRepository.list(zone.getId(), NodeFilter.all()).stream())) .filter(node -> node.owner().isPresent() && !node.owner().get().tenant().equals(SystemApplication.TENANT)) .collect(Collectors.toList()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 63f33540721..670cb775c69 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -12,6 +12,7 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; import com.yahoo.component.annotation.Inject; import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -90,7 +91,6 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; @@ -137,10 +137,13 @@ import java.security.PublicKey; import java.time.DayOfWeek; import java.time.Duration; import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Base64; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -158,7 +161,9 @@ import java.util.stream.Stream; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.CONFLICT; import static com.yahoo.yolean.Exceptions.uncheck; +import static java.util.Comparator.comparingInt; import static java.util.Map.Entry.comparingByKey; +import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toUnmodifiableList; @@ -280,6 +285,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/access/support")) return supportAccess(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/node/{node}/service-dump")) return getServiceDump(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("node"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/scaling")) return scaling(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/metrics")) return metrics(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("endpointId"))); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); @@ -1423,6 +1429,29 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return buildResponseFromProtonMetrics(protonMetrics); } + private HttpResponse scaling(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { + var from = Optional.ofNullable(request.getProperty("from")) + .map(Long::valueOf) + .map(Instant::ofEpochSecond) + .orElse(Instant.EPOCH); + var until = Optional.ofNullable(request.getProperty("until")) + .map(Long::valueOf) + .map(Instant::ofEpochSecond) + .orElse(Instant.now(controller.clock())); + + var application = ApplicationId.from(tenantName, applicationName, instanceName); + var zone = requireZone(environment, region); + var deployment = new DeploymentId(application, zone); + var events = controller.serviceRegistry().resourceDatabase().scalingEvents(from, until, deployment); + var slime = new Slime(); + var root = slime.setObject(); + for (var entry : events.entrySet()) { + var serviceRoot = root.setArray(entry.getKey().clusterId().value()); + scalingEventsToSlime(entry.getValue(), serviceRoot); + } + return new SlimeJsonResponse(slime); + } + private JsonResponse buildResponseFromProtonMetrics(List<ProtonMetrics> protonMetrics) { try { var jsonObject = jsonMapper.createObjectNode(); @@ -1528,10 +1557,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { object.setString("instance", instance.name().value()); if (deploymentSpec.instance(instance.name()).isPresent()) { - // Jobs sorted according to deployment spec - List<JobStatus> jobStatus = controller.applications().deploymentTrigger() - .steps(deploymentSpec.requireInstance(instance.name())) - .sortedJobs(status.instanceJobs(instance.name()).values()); + // Jobs ordered according to deployment spec + Collection<JobStatus> jobStatus = status.instanceJobs(instance.name()).values(); if ( ! instance.change().isEmpty()) toSlime(object.setObject("deploying"), instance.change(), status.application()); @@ -1559,8 +1586,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { // Deployments sorted according to deployment spec List<Deployment> deployments = deploymentSpec.instance(instance.name()) - .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry())) - .map(steps -> steps.sortedDeployments(instance.deployments().values())) + .map(spec -> sortedDeployments(instance.deployments().values(), spec)) .orElse(List.copyOf(instance.deployments().values())); Cursor deploymentsArray = object.setArray("deployments"); @@ -1613,10 +1639,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { application.projectId().ifPresent(id -> object.setLong("projectId", id)); if (application.deploymentSpec().instance(instance.name()).isPresent()) { - // Jobs sorted according to deployment spec - List<JobStatus> jobStatus = controller.applications().deploymentTrigger() - .steps(application.deploymentSpec().requireInstance(instance.name())) - .sortedJobs(status.instanceJobs(instance.name()).values()); + // Jobs ordered according to deployment spec + Collection<JobStatus> jobStatus = status.instanceJobs(instance.name()).values(); if ( ! instance.change().isEmpty()) toSlime(object.setObject("deploying"), instance.change(), application); @@ -1645,11 +1669,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { addRotationId(object, instance); // Deployments sorted according to deployment spec - List<Deployment> deployments = - application.deploymentSpec().instance(instance.name()) - .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry())) - .map(steps -> steps.sortedDeployments(instance.deployments().values())) - .orElse(List.copyOf(instance.deployments().values())); + List<Deployment> deployments = application.deploymentSpec().instance(instance.name()) + .map(spec -> sortedDeployments(instance.deployments().values(), spec)) + .orElse(List.copyOf(instance.deployments().values())); Cursor instancesArray = object.setArray("instances"); for (Deployment deployment : deployments) { Cursor deploymentObject = instancesArray.addObject(); @@ -3017,5 +3039,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } } + private List<Deployment> sortedDeployments(Collection<Deployment> deployments, DeploymentInstanceSpec spec) { + List<ZoneId> productionZones = spec.zones().stream() + .filter(z -> z.region().isPresent()) + .map(z -> ZoneId.from(z.environment(), z.region().get())) + .toList(); + return deployments.stream() + .sorted(comparingInt(deployment -> productionZones.indexOf(deployment.zone()))) + .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 8c49df43c30..9c016eccd27 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -151,13 +151,7 @@ class JobControllerApiHandlerHelper { Optional<String> testReport = jobController.getTestReports(runId); testReport.map(SlimeUtils::jsonToSlime) .map(Slime::get) - .ifPresent(reportArrayCursor -> { - reportArrayCursor.traverse((ArrayTraverser) (i, reportCursor) -> { - if (i > 0) return; - SlimeUtils.copyObject(reportCursor, detailsObject.setObject("testReport")); - }); - SlimeUtils.copyArray(reportArrayCursor, detailsObject.setArray("testReports")); - }); + .ifPresent(reportArrayCursor -> SlimeUtils.copyArray(reportArrayCursor, detailsObject.setArray("testReports"))); return new SlimeJsonResponse(slime); } @@ -331,7 +325,7 @@ class JobControllerApiHandlerHelper { "/job/" + job.type().jobName()).normalize(); stepObject.setString("url", baseUriForJob.toString()); stepObject.setString("environment", job.type().environment().value()); - stepObject.setString("region", job.type().zone().value()); + if ( ! job.type().environment().isTest()) stepObject.setString("region", job.type().zone().value()); if (job.type().isProduction() && job.type().isDeployment()) { status.deploymentFor(job).ifPresent(deployment -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index b80a487ec50..44a8b636ae0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.TenantController; import com.yahoo.vespa.hosted.controller.api.integration.billing.Bill; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.CollectionMethod; +import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.api.role.Role; @@ -91,16 +92,14 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); var tenant = tenants.require(tenantName, CloudTenant.class); - var plan = planRegistry.plan(billing.getPlan(tenant.name())).orElseThrow(); + var plan = planFor(tenant.name()); var collectionMethod = billing.getCollectionMethod(tenant.name()); var response = new Slime(); var cursor = response.setObject(); cursor.setString("tenant", tenant.name().value()); - var planCursor = cursor.setObject("plan"); - planCursor.setString("id", plan.id().value()); - planCursor.setString("name", plan.displayName()); + toSlime(cursor.setObject("plan"), plan); cursor.setString("collection", collectionMethod.name()); return response; } @@ -137,7 +136,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler var response = new Slime(); var cursor = response.setObject(); cursor.setString("tenant", tenant.name().value()); - cursor.setString("plan", billing.getPlan(tenant.name()).value()); + toSlime(cursor.setObject("plan"), planFor(tenant.name())); cursor.setString("collection", billing.getCollectionMethod(tenant.name()).name()); return response; } @@ -205,7 +204,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler var usage = Optional.ofNullable(usagePerTenant.get(tenant.name())); var tenantResponse = tenantsResponse.addObject(); tenantResponse.setString("tenant", tenant.name().value()); - tenantResponse.setString("plan", billing.getPlan(tenant.name()).value()); + toSlime(tenantResponse.setObject("plan"), planFor(tenant.name())); tenantResponse.setString("collection", billing.getCollectionMethod(tenant.name()).name()); tenantResponse.setString("lastBill", usage.map(Bill::getStartDate).map(DateTimeFormatter.ISO_DATE::format).orElse(null)); tenantResponse.setString("unbilled", usage.map(Bill::sum).map(BigDecimal::toPlainString).orElse("0.00")); @@ -304,8 +303,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler slime.setString("id", item.id()); slime.setString("description", item.description()); slime.setString("amount",item.amount().toString()); - slime.setString("plan", item.plan()); - slime.setString("planName", billing.getPlanDisplayName(PlanId.from(item.plan()))); + toSlime(slime.setObject("plan"), planRegistry.plan(item.plan()).orElseThrow(() -> new RuntimeException("No such plan: '" + item.plan() + "'"))); item.applicationId().ifPresent(appId -> { slime.setString("application", appId.application().value()); @@ -354,4 +352,14 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return inspector.field(field).asString(); } + private void toSlime(Cursor cursor, Plan plan) { + cursor.setString("id", plan.id().value()); + cursor.setString("name", plan.displayName()); + } + + private Plan planFor(TenantName tenant) { + var planId = billing.getPlan(tenant); + return planRegistry.plan(planId) + .orElseThrow(() -> new RuntimeException("No such plan: '" + planId + "'")); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 3ca6335dcb7..25ac90ac0ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -84,7 +84,7 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(controller.jobControl()); if (path.matches("/controller/v1/stats")) return new StatsResponse(controller); if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader()); - if (path.matches("/controller/v1/metering/tenant/{tenant}/month/{month}")) return new MeteringResponse(controller.serviceRegistry().meteringService(), path.get("tenant"), path.get("month")); + if (path.matches("/controller/v1/metering/tenant/{tenant}/month/{month}")) return new MeteringResponse(controller.serviceRegistry().resourceDatabase(), path.get("tenant"), path.get("month")); return notFound(path); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java index 33cd4948a7e..17461aafd02 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java @@ -5,7 +5,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import java.time.YearMonth; @@ -16,14 +16,14 @@ import java.util.List; */ public class MeteringResponse extends SlimeJsonResponse { - public MeteringResponse(MeteringClient meteringClient, String tenantName, String month) { - super(toSlime(meteringClient, tenantName, month)); + public MeteringResponse(ResourceDatabaseClient resourceClient, String tenantName, String month) { + super(toSlime(resourceClient, tenantName, month)); } - private static Slime toSlime(MeteringClient meteringClient, String tenantName, String month) { + private static Slime toSlime(ResourceDatabaseClient resourceClient, String tenantName, String month) { Slime slime = new Slime(); Cursor root = slime.setArray(); - List<ResourceSnapshot> snapshots = meteringClient.getSnapshotHistoryForTenant(TenantName.from(tenantName), YearMonth.parse(month)); + List<ResourceSnapshot> snapshots = resourceClient.getRawSnapshotHistoryForTenant(TenantName.from(tenantName), YearMonth.parse(month)); snapshots.forEach(snapshot -> { Cursor object = root.addObject(); object.setString("applicationId", snapshot.getApplicationId().toShortString()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 8ce55fa5f1b..28536f36e20 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -748,15 +748,15 @@ public class ControllerTest { fail("Should fail when specifying a major that does not yet exist"); } catch (IllegalArgumentException e) { - assertEquals("major 8 specified in deployment.xml, but no version on this major was found", e.getMessage()); + assertEquals("no platforms were found for major version 8 specified in deployment.xml", e.getMessage()); } try { context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version3).build()); - fail("Should fail when compiled against a version which does not yet exist"); + fail("Should fail when compiled against a version which is only compatible with not-yet-existent versions"); } catch (IllegalArgumentException e) { - assertEquals("no suitable platform version found for package compiled against 8", e.getMessage()); + assertEquals("no platforms are compatible with compile version 8", e.getMessage()); } tester.controllerTester().upgradeSystem(version3); @@ -765,7 +765,7 @@ public class ControllerTest { fail("Should fail when specifying a major which is incompatible with compile version"); } catch (IllegalArgumentException e) { - assertEquals("Will not start dev-us-east-1 for tenant.application with incompatible platform version (8) and compile versions (7)", e.getMessage()); + assertEquals("no platforms on major version 8 specified in deployment.xml are compatible with compile version 7", e.getMessage()); } context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version3).majorVersion(8).build()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java index 96b0e3137cb..11b3ad0d45a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java @@ -101,7 +101,7 @@ public class DeploymentQuotaCalculatorTest { var mapper = new ObjectMapper(); var application = mapper.readValue(content, ApplicationData.class).toApplication(); var usage = DeploymentQuotaCalculator.calculateQuotaUsage(application); - assertEquals(1.068, usage.rate(), 0.001); + assertEquals(1.312, usage.rate(), 0.001); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index 59aafe73fc1..4b6d802260e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -108,7 +108,7 @@ public class EndpointCertificatesTest { @Before public void setUp() { tester.zoneRegistry().exclusiveRoutingIn(tester.zoneRegistry().zones().all().zones()); - testZone = tester.zoneRegistry().zones().routingMethod(RoutingMethod.exclusive).in(Environment.prod).zones().stream().findFirst().orElseThrow().getId(); + testZone = tester.zoneRegistry().zones().all().routingMethod(RoutingMethod.exclusive).in(Environment.prod).zones().stream().findFirst().orElseThrow().getId(); clock.setInstant(Instant.EPOCH); testCertificate = makeTestCert(expectedSans); testCertificate2 = makeTestCert(expectedCombinedSans); @@ -116,7 +116,7 @@ public class EndpointCertificatesTest { @Test public void provisions_new_certificate_in_dev() { - ZoneId testZone = tester.zoneRegistry().zones().routingMethod(RoutingMethod.exclusive).in(Environment.dev).zones().stream().findFirst().orElseThrow().getId(); + ZoneId testZone = tester.zoneRegistry().zones().all().routingMethod(RoutingMethod.exclusive).in(Environment.dev).zones().stream().findFirst().orElseThrow().getId(); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificates.getMetadata(testInstance, testZone, DeploymentSpec.empty); assertTrue(endpointCertificateMetadata.isPresent()); assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); @@ -190,7 +190,7 @@ public class EndpointCertificatesTest { @Test public void reprovisions_certificate_with_added_sans_when_deploying_to_new_zone() { - ZoneId testZone = tester.zoneRegistry().zones().routingMethod(RoutingMethod.exclusive).in(Environment.prod).zones().stream().skip(1).findFirst().orElseThrow().getId(); + ZoneId testZone = tester.zoneRegistry().zones().all().routingMethod(RoutingMethod.exclusive).in(Environment.prod).zones().stream().skip(1).findFirst().orElseThrow().getId(); mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "original-request-uuid", Optional.of("leaf-request-uuid"), expectedSans, "mockCa", Optional.empty(), Optional.empty())); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), -1); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index b3db4a8b845..ad236fcd4de 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -220,12 +220,11 @@ public class DeploymentContext { .allMatch(deployments -> deployments.stream() .allMatch(deployment -> deployment.version().equals(version)))); - for (var spec : application().deploymentSpec().instances()) - for (JobType type : new DeploymentSteps(spec, tester.controller().zoneRegistry()).productionJobs()) - assertTrue(tester.configServer().nodeRepository() - .list(type.zone(), - NodeFilter.all().applications(applicationId.defaultInstance())).stream() - .allMatch(node -> node.currentVersion().equals(version))); + for (JobId job : deploymentStatus().jobs().matching(job -> job.id().type().isProduction()).mapToList(JobStatus::id)) + assertTrue(tester.configServer().nodeRepository() + .list(job.type().zone(), + NodeFilter.all().applications(job.application())).stream() + .allMatch(node -> node.currentVersion().equals(version))); assertFalse(instance().change().hasTargets()); return this; @@ -385,13 +384,13 @@ public class DeploymentContext { /** Runs and returns all remaining jobs for the application, at most once, and asserts the current change is rolled out. */ public DeploymentContext completeRollout(boolean multiInstance) { triggerJobs(); - Map<ApplicationId, Map<JobType, Versions>> jobsByInstance = new HashMap<>(); + Map<ApplicationId, Map<JobType, Run>> runsByInstance = new HashMap<>(); List<Run> activeRuns; while ( ! (activeRuns = this.jobs.active(applicationId)).isEmpty()) for (Run run : activeRuns) { - Map<JobType, Versions> jobs = jobsByInstance.computeIfAbsent(run.id().application(), k -> new HashMap<>()); - Versions previous = jobs.put(run.id().type(), run.versions()); - if (run.versions().equals(previous)) { + Map<JobType, Run> runs = runsByInstance.computeIfAbsent(run.id().application(), k -> new HashMap<>()); + Run previous = runs.put(run.id().type(), run); + if (previous != null && run.versions().equals(previous.versions()) && run.id().type().zone().equals(previous.id().type().zone())) { throw new AssertionError("Job '" + run.id() + "' was run twice on same versions"); } runJob(run.id().type(), run.id().application()); @@ -452,8 +451,8 @@ public class DeploymentContext { /** Pulls the ready job trigger, and then runs the whole of job for the given instance, successfully. */ private DeploymentContext runJob(JobType type, ApplicationId instance) { - var job = new JobId(instance, type); triggerJobs(); + var job = currentRun(new JobId(instance, type)).id().job(); doDeploy(job); if (job.type().isDeployment()) { doUpgrade(job); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 67ddc767b39..673cbf9708b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -2,23 +2,34 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Assert; import org.junit.Test; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.List; @@ -53,6 +64,7 @@ import static java.util.Collections.emptyList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; /** @@ -981,36 +993,38 @@ public class DeploymentTriggerTest { DeploymentContext i3 = tester.newDeploymentContext("t", "a", "i3"); DeploymentContext i4 = tester.newDeploymentContext("t", "a", "i4"); ApplicationPackage applicationPackage = ApplicationPackageBuilder - .fromDeploymentXml("<deployment version='1'>\n" + - " <upgrade revision-change='when-failing' />\n" + - " <parallel>\n" + - " <instance id='i1'>\n" + - " <prod>\n" + - " <region>us-east-3</region>\n" + - " <delay hours='6' />\n" + - " </prod>\n" + - " </instance>\n" + - " <instance id='i2'>\n" + - " <prod>\n" + - " <region>us-east-3</region>\n" + - " </prod>\n" + - " </instance>\n" + - " </parallel>\n" + - " <instance id='i3'>\n" + - " <prod>\n" + - " <region>us-east-3</region>\n" + - " <delay hours='18' />\n" + - " <test>us-east-3</test>\n" + - " </prod>\n" + - " </instance>\n" + - " <instance id='i4'>\n" + - " <test />\n" + - " <staging />\n" + - " <prod>\n" + - " <region>us-east-3</region>\n" + - " </prod>\n" + - " </instance>\n" + - "</deployment>\n"); + .fromDeploymentXml(""" + <deployment version='1'> + <upgrade revision-change='when-failing' /> + <parallel> + <instance id='i1'> + <prod> + <region>us-east-3</region> + <delay hours='6' /> + </prod> + </instance> + <instance id='i2'> + <prod> + <region>us-east-3</region> + </prod> + </instance> + </parallel> + <instance id='i3'> + <prod> + <region>us-east-3</region> + <delay hours='18' /> + <test>us-east-3</test> + </prod> + </instance> + <instance id='i4'> + <test /> + <staging /> + <prod> + <region>us-east-3</region> + </prod> + </instance> + </deployment> + """); // Package is submitted, and change propagated to the two first instances. i1.submit(applicationPackage); @@ -1085,20 +1099,22 @@ public class DeploymentTriggerTest { @Test public void testMultipleInstancesWithRevisionCatchingUpToUpgrade() { - String spec = "<deployment>\n" + - " <instance id='alpha'>\n" + - " <upgrade rollout=\"simultaneous\" revision-target=\"next\" />\n" + - " <test />\n" + - " <staging />\n" + - " </instance>\n" + - " <instance id='beta'>\n" + - " <upgrade rollout=\"simultaneous\" revision-change=\"when-clear\" revision-target=\"next\" />\n" + - " <prod>\n" + - " <region>us-east-3</region>\n" + - " <test>us-east-3</test>\n" + - " </prod>\n" + - " </instance>\n" + - "</deployment>\n"; + String spec = """ + <deployment> + <instance id='alpha'> + <upgrade rollout="simultaneous" revision-target="next" /> + <test /> + <staging /> + </instance> + <instance id='beta'> + <upgrade rollout="simultaneous" revision-change="when-clear" revision-target="next" /> + <prod> + <region>us-east-3</region> + <test>us-east-3</test> + </prod> + </instance> + </deployment> + """; ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(spec); DeploymentContext alpha = tester.newDeploymentContext("t", "a", "alpha"); DeploymentContext beta = tester.newDeploymentContext("t", "a", "beta"); @@ -1236,67 +1252,69 @@ public class DeploymentTriggerTest { @Test public void testDeployComplicatedDeploymentSpec() { String complicatedDeploymentSpec = - "<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" + - " <parallel>\n" + - " <instance id='instance' athenz-service='in-service'>\n" + - " <staging />\n" + - " <prod>\n" + - " <parallel>\n" + - " <region active='true'>us-west-1</region>\n" + - " <steps>\n" + - " <region active='true'>us-east-3</region>\n" + - " <delay hours='2' />\n" + - " <region active='true'>eu-west-1</region>\n" + - " <delay hours='2' />\n" + - " </steps>\n" + - " <steps>\n" + - " <delay hours='3' />\n" + - " <region active='true'>aws-us-east-1a</region>\n" + - " <parallel>\n" + - " <region active='true' athenz-service='no-service'>ap-northeast-1</region>\n" + - " <region active='true'>ap-northeast-2</region>\n" + - " <test>aws-us-east-1a</test>\n" + - " </parallel>\n" + - " </steps>\n" + - " <delay hours='3' minutes='30' />\n" + - " </parallel>\n" + - " <parallel>\n" + - " <test>ap-northeast-2</test>\n" + - " <test>ap-northeast-1</test>\n" + - " </parallel>\n" + - " <test>us-east-3</test>\n" + - " <region active='true'>ap-southeast-1</region>\n" + - " </prod>\n" + - " <endpoints>\n" + - " <endpoint id='foo' container-id='bar'>\n" + - " <region>us-east-3</region>\n" + - " </endpoint>\n" + - " <endpoint id='nalle' container-id='frosk' />\n" + - " <endpoint container-id='quux' />\n" + - " </endpoints>\n" + - " </instance>\n" + - " <instance id='other'>\n" + - " <upgrade policy='conservative' />\n" + - " <test />\n" + - " <block-change revision='true' version='false' days='sat' hours='0-23' time-zone='CET' />\n" + - " <prod>\n" + - " <region active='true'>eu-west-1</region>\n" + - " <test>eu-west-1</test>\n" + - " </prod>\n" + - " <notifications when='failing'>\n" + - " <email role='author' />\n" + - " <email address='john@dev' when='failing-commit' />\n" + - " <email address='jane@dev' />\n" + - " </notifications>\n" + - " </instance>\n" + - " </parallel>\n" + - " <instance id='last'>\n" + - " <upgrade policy='conservative' />\n" + - " <prod>\n" + - " <region active='true'>eu-west-1</region>\n" + - " </prod>\n" + - " </instance>\n" + - "</deployment>\n"; + """ + <deployment version='1.0' athenz-domain='domain' athenz-service='service'> + <parallel> + <instance id='instance' athenz-service='in-service'> + <staging /> + <prod> + <parallel> + <region active='true'>us-west-1</region> + <steps> + <region active='true'>us-east-3</region> + <delay hours='2' /> + <region active='true'>eu-west-1</region> + <delay hours='2' /> + </steps> + <steps> + <delay hours='3' /> + <region active='true'>aws-us-east-1a</region> + <parallel> + <region active='true' athenz-service='no-service'>ap-northeast-1</region> + <region active='true'>ap-northeast-2</region> + <test>aws-us-east-1a</test> + </parallel> + </steps> + <delay hours='3' minutes='30' /> + </parallel> + <parallel> + <test>ap-northeast-2</test> + <test>ap-northeast-1</test> + </parallel> + <test>us-east-3</test> + <region active='true'>ap-southeast-1</region> + </prod> + <endpoints> + <endpoint id='foo' container-id='bar'> + <region>us-east-3</region> + </endpoint> + <endpoint id='nalle' container-id='frosk' /> + <endpoint container-id='quux' /> + </endpoints> + </instance> + <instance id='other'> + <upgrade policy='conservative' /> + <test /> + <block-change revision='true' version='false' days='sat' hours='0-23' time-zone='CET' /> + <prod> + <region active='true'>eu-west-1</region> + <test>eu-west-1</test> + </prod> + <notifications when='failing'> + <email role='author' /> + <email address='john@dev' when='failing-commit' /> + <email address='jane@dev' /> + </notifications> + </instance> + </parallel> + <instance id='last'> + <upgrade policy='conservative' /> + <prod> + <region active='true'>eu-west-1</region> + </prod> + </instance> + </deployment> + """; tester.atMondayMorning(); ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(complicatedDeploymentSpec); @@ -1640,31 +1658,33 @@ public class DeploymentTriggerTest { @Test public void testVeryLengthyPipelineRevisions() { String lengthyDeploymentSpec = - "<deployment version='1.0'>\n" + - " <instance id='alpha'>\n" + - " <test />\n" + - " <staging />\n" + - " <upgrade revision-change='always' />\n" + - " <prod>\n" + - " <region>us-east-3</region>\n" + - " <test>us-east-3</test>\n" + - " </prod>\n" + - " </instance>\n" + - " <instance id='beta'>\n" + - " <upgrade revision-change='when-failing' />\n" + - " <prod>\n" + - " <region>us-east-3</region>\n" + - " <test>us-east-3</test>\n" + - " </prod>\n" + - " </instance>\n" + - " <instance id='gamma'>\n" + - " <upgrade revision-change='when-clear' revision-target='next' min-risk='3' max-risk='6' />\n" + - " <prod>\n" + - " <region>us-east-3</region>\n" + - " <test>us-east-3</test>\n" + - " </prod>\n" + - " </instance>\n" + - "</deployment>\n"; + """ + <deployment version='1.0'> + <instance id='alpha'> + <test /> + <staging /> + <upgrade revision-change='always' /> + <prod> + <region>us-east-3</region> + <test>us-east-3</test> + </prod> + </instance> + <instance id='beta'> + <upgrade revision-change='when-failing' /> + <prod> + <region>us-east-3</region> + <test>us-east-3</test> + </prod> + </instance> + <instance id='gamma'> + <upgrade revision-change='when-clear' revision-target='next' min-risk='3' max-risk='6' /> + <prod> + <region>us-east-3</region> + <test>us-east-3</test> + </prod> + </instance> + </deployment> + """; var appPackage = ApplicationPackageBuilder.fromDeploymentXml(lengthyDeploymentSpec); var alpha = tester.newDeploymentContext("t", "a", "alpha"); var beta = tester.newDeploymentContext("t", "a", "beta"); @@ -1783,31 +1803,33 @@ public class DeploymentTriggerTest { @Test public void testVeryLengthyPipelineUpgrade() { String lengthyDeploymentSpec = - "<deployment version='1.0'>\n" + - " <instance id='alpha'>\n" + - " <test />\n" + - " <staging />\n" + - " <upgrade rollout='simultaneous' />\n" + - " <prod>\n" + - " <region>us-east-3</region>\n" + - " <test>us-east-3</test>\n" + - " </prod>\n" + - " </instance>\n" + - " <instance id='beta'>\n" + - " <upgrade rollout='simultaneous' />\n" + - " <prod>\n" + - " <region>us-east-3</region>\n" + - " <test>us-east-3</test>\n" + - " </prod>\n" + - " </instance>\n" + - " <instance id='gamma'>\n" + - " <upgrade rollout='separate' />\n" + - " <prod>\n" + - " <region>us-east-3</region>\n" + - " <test>us-east-3</test>\n" + - " </prod>\n" + - " </instance>\n" + - "</deployment>\n"; + """ + <deployment version='1.0'> + <instance id='alpha'> + <test /> + <staging /> + <upgrade rollout='simultaneous' /> + <prod> + <region>us-east-3</region> + <test>us-east-3</test> + </prod> + </instance> + <instance id='beta'> + <upgrade rollout='simultaneous' /> + <prod> + <region>us-east-3</region> + <test>us-east-3</test> + </prod> + </instance> + <instance id='gamma'> + <upgrade rollout='separate' /> + <prod> + <region>us-east-3</region> + <test>us-east-3</test> + </prod> + </instance> + </deployment> + """; var appPackage = ApplicationPackageBuilder.fromDeploymentXml(lengthyDeploymentSpec); var alpha = tester.newDeploymentContext("t", "a", "alpha"); var beta = tester.newDeploymentContext("t", "a", "beta"); @@ -1986,19 +2008,21 @@ public class DeploymentTriggerTest { @Test public void testsInSeparateInstance() { String deploymentSpec = - "<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" + - " <instance id='canary'>\n" + - " <upgrade policy='canary' />\n" + - " <test />\n" + - " <staging />\n" + - " </instance>\n" + - " <instance id='default'>\n" + - " <prod>\n" + - " <region>eu-west-1</region>\n" + - " <test>eu-west-1</test>\n" + - " </prod>\n" + - " </instance>\n" + - "</deployment>\n"; + """ + <deployment version='1.0' athenz-domain='domain' athenz-service='service'> + <instance id='canary'> + <upgrade policy='canary' /> + <test /> + <staging /> + </instance> + <instance id='default'> + <prod> + <region>eu-west-1</region> + <test>eu-west-1</test> + </prod> + </instance> + </deployment> + """; ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(deploymentSpec); var canary = tester.newDeploymentContext("t", "a", "canary").submit(applicationPackage); @@ -2157,29 +2181,60 @@ public class DeploymentTriggerTest { } @Test - public void testInstanceWithOnlySystemTest() { - String spec = "<deployment>\n" + - " <instance id='tests'>" + - " <test />\n" + - " <upgrade revision-target='next' />" + - " </instance>\n" + - " <instance id='main'>\n" + - " <prod>\n" + - " <region>us-east-3</region>\n" + - " </prod>\n" + - " <upgrade revision-target='next' />" + - " </instance>\n" + - "</deployment>\n"; + public void testInstanceWithOnlySystemTestInTwoClouds() { + String spec = """ + <deployment> + <instance id='tests'> + <test /> + <upgrade revision-target='next' /> + </instance> + <instance id='main'> + <prod> + <region>us-east-3</region> + <region>alpha-centauri</region> + </prod> + <upgrade revision-target='next' /> + </instance> + </deployment> + """; + + RegionName alphaCentauri = RegionName.from("alpha-centauri"); + ZoneApiMock.Builder builder = ZoneApiMock.newBuilder().withCloud("centauri").withSystem(tester.controller().system()); + ZoneApi testAlphaCentauri = builder.with(ZoneId.from(Environment.test, alphaCentauri)).build(); + ZoneApi stagingAlphaCentauri = builder.with(ZoneId.from(Environment.staging, alphaCentauri)).build(); + ZoneApi prodAlphaCentauri = builder.with(ZoneId.from(Environment.prod, alphaCentauri)).build(); + + tester.controllerTester().zoneRegistry().addZones(testAlphaCentauri, stagingAlphaCentauri, prodAlphaCentauri); + tester.controllerTester().setRoutingMethod(tester.controllerTester().zoneRegistry().zones().all().ids(), RoutingMethod.sharedLayer4); + tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.notController()); + ApplicationPackage appPackage = ApplicationPackageBuilder.fromDeploymentXml(spec); DeploymentContext tests = tester.newDeploymentContext("tenant", "application", "tests"); DeploymentContext main = tester.newDeploymentContext("tenant", "application", "main"); Version version1 = new Version("7"); tester.controllerTester().upgradeSystem(version1); - tests.submit(appPackage).deploy(); + tests.submit(appPackage); Optional<RevisionId> revision1 = tests.lastSubmission(); JobId systemTestJob = new JobId(tests.instanceId(), systemTest); JobId stagingTestJob = new JobId(tests.instanceId(), stagingTest); JobId mainJob = new JobId(main.instanceId(), productionUsEast3); + JobId centauriJob = new JobId(main.instanceId(), JobType.deploymentTo(prodAlphaCentauri.getId())); + + assertEquals(Set.of(systemTestJob, stagingTestJob, mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet()); + tests.runJob(systemTest).runJob(stagingTest).triggerJobs(); + + assertEquals(Set.of(systemTestJob, stagingTestJob, mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet()); + tests.triggerJobs(); + assertEquals(3, tester.jobs().active().size()); + + tests.runJob(systemTest); + tester.outstandingChangeDeployer().run(); + + assertEquals(2, tester.jobs().active().size()); + main.assertRunning(productionUsEast3); + + tests.runJob(stagingTest); + main.runJob(productionUsEast3).runJob(centauriJob.type()); assertEquals(Change.empty(), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); @@ -2197,25 +2252,28 @@ public class DeploymentTriggerTest { assertEquals(Change.of(version2), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); + assertEquals(2, tests.deploymentStatus().jobsToRun().get(systemTestJob).size()); Version version3 = new Version("9"); tester.controllerTester().upgradeSystem(version3); - tests.failDeployment(systemTest); + tests.runJob(systemTest) // Success in default cloud. + .failDeployment(systemTest); // Failure in centauri cloud. tester.upgrader().run(); assertEquals(Change.of(version3), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); - tests.runJob(systemTest); + tests.runJob(systemTest).runJob(systemTest); tester.upgrader().run(); - tests.runJob(stagingTest); + tests.runJob(stagingTest).runJob(stagingTest); assertEquals(Change.empty(), tests.instance().change()); assertEquals(Change.of(version3), main.instance().change()); - assertEquals(Set.of(mainJob), tests.deploymentStatus().jobsToRun().keySet()); + assertEquals(Set.of(mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet()); main.runJob(productionUsEast3); + main.runJob(centauriJob.type()); assertEquals(Change.empty(), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); @@ -2237,6 +2295,7 @@ public class DeploymentTriggerTest { assertEquals(Change.of(revision2.get()), tests.instance().change()); assertEquals(Change.empty(), main.instance().change()); assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); + assertEquals(2, tests.deploymentStatus().jobsToRun().get(systemTestJob).size()); tests.submit(appPackage); Optional<RevisionId> revision3 = tests.lastSubmission(); @@ -2253,15 +2312,34 @@ public class DeploymentTriggerTest { assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet()); tests.runJob(systemTest); + assertEquals(Change.of(revision3.get()), tests.instance().change()); + assertEquals(Change.empty(), main.instance().change()); + assertEquals(Set.of(systemTestJob, stagingTestJob), tests.deploymentStatus().jobsToRun().keySet()); + tester.outstandingChangeDeployer().run(); tester.outstandingChangeDeployer().run(); tests.runJob(stagingTest); + assertEquals(Change.of(revision3.get()), tests.instance().change()); + assertEquals(Change.empty(), main.instance().change()); + assertEquals(Set.of(systemTestJob, stagingTestJob), tests.deploymentStatus().jobsToRun().keySet()); + + tests.runJob(systemTest); + tester.outstandingChangeDeployer().run(); + tester.outstandingChangeDeployer().run(); + + assertEquals(Change.empty(), tests.instance().change()); + assertEquals(Change.of(revision3.get()), main.instance().change()); + assertEquals(Set.of(stagingTestJob, mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet()); + + tests.runJob(stagingTest); + assertEquals(Change.empty(), tests.instance().change()); assertEquals(Change.of(revision3.get()), main.instance().change()); - assertEquals(Set.of(mainJob), tests.deploymentStatus().jobsToRun().keySet()); + assertEquals(Set.of(mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet()); main.runJob(productionUsEast3); + main.runJob(centauriJob.type()); tester.outstandingChangeDeployer().run(); assertEquals(Change.empty(), tests.instance().change()); @@ -2287,4 +2365,32 @@ public class DeploymentTriggerTest { assertTrue(tester.jobs().last(app.instanceId(), stagingTest).get().hasSucceeded()); } + @Test + public void testJobNames() { + ZoneRegistryMock zones = new ZoneRegistryMock(SystemName.main); + List<ZoneApi> existing = new ArrayList<>(zones.zones().all().zones()); + existing.add(ZoneApiMock.newBuilder().withCloud("pink-clouds").withId("test.zone").build()); + zones.setZones(existing); + + JobType defaultSystemTest = JobType.systemTest(zones, CloudName.defaultName()); + JobType pinkSystemTest = JobType.systemTest(zones, CloudName.from("pink-clouds")); + + // Job name is identity, used for looking up run history, etc.. + assertEquals(defaultSystemTest, pinkSystemTest); + + assertEquals(defaultSystemTest, JobType.systemTest(zones, null)); + assertEquals(defaultSystemTest, JobType.systemTest(zones, CloudName.from("dark-clouds"))); + assertEquals(defaultSystemTest, JobType.fromJobName("system-test", zones)); + + assertEquals(ZoneId.from("test", "us-east-1"), defaultSystemTest.zone()); + assertEquals(ZoneId.from("staging", "us-east-3"), JobType.stagingTest(zones, zones.systemZone().getCloudName()).zone()); + + assertEquals(ZoneId.from("test", "zone"), pinkSystemTest.zone()); + assertEquals(ZoneId.from("staging", "us-east-3"), JobType.stagingTest(zones, CloudName.from("pink-clouds")).zone()); + + assertThrows(IllegalStateException.class, JobType.systemTest(zones, null)::zone); + assertThrows(IllegalStateException.class, JobType.fromJobName("system-test", zones)::zone); + assertThrows(IllegalStateException.class, JobType.fromJobName("staging-test", zones)::zone); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/QuotaUsageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/QuotaUsageTest.java index 5ce6c95ee82..772d65fa80e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/QuotaUsageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/QuotaUsageTest.java @@ -15,7 +15,7 @@ public class QuotaUsageTest { public void testQuotaUsageIsPersisted() { var tester = new DeploymentTester(); var context = tester.newDeploymentContext().submit().deploy(); - assertEquals(1.062, context.deployment(ZoneId.from("prod.us-west-1")).quota().rate(), 0.01); + assertEquals(1.304, context.deployment(ZoneId.from("prod.us-west-1")).quota().rate(), 0.01); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 8e0d70bdb80..af542521b31 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -44,7 +44,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIss import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummySystemMonitor; import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; -import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; @@ -69,7 +68,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final MockMailer mockMailer = new MockMailer(); private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(clock); private final EndpointCertificateValidatorMock endpointCertificateValidatorMock = new EndpointCertificateValidatorMock(); - private final MockMeteringClient mockMeteringClient = new MockMeteringClient(); private final MockContactRetriever mockContactRetriever = new MockContactRetriever(); private final MockIssueHandler mockIssueHandler = new MockIssueHandler(); private final DummyOwnershipIssues dummyOwnershipIssues = new DummyOwnershipIssues(); @@ -148,11 +146,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override - public MockMeteringClient meteringService() { - return mockMeteringClient; - } - - @Override public MockContactRetriever contactRetriever() { return mockContactRetriever; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index c17b2fcb36a..c87ac229c4b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -81,6 +81,11 @@ public class ZoneFilterMock implements ZoneList { } @Override + public ZoneList in(CloudName... clouds) { + return filter(zone -> Set.of(clouds).contains(zone.getCloudName())); + } + + @Override public ZoneList among(ZoneId... zones) { return filter(zone -> Set.of(zones).contains(zone.getId())); } @@ -90,11 +95,6 @@ public class ZoneFilterMock implements ZoneList { return List.copyOf(zones); } - @Override - public ZoneList ofCloud(CloudName cloud) { - return filter(zone -> zone.getCloudName().equals(cloud)); - } - private ZoneFilterMock filter(Predicate<ZoneApi> condition) { return new ZoneFilterMock( zones.stream() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 849503ae8d1..4131bfd8b9f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -26,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -45,8 +46,6 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final Set<ZoneApi> reprovisionToUpgradeOs = new HashSet<>(); private final SystemName system; // Don't even think about making it non-final! ƪ(`▿▿▿▿´ƪ) - - private List<? extends ZoneApi> zones; private UpgradePolicy upgradePolicy = null; @@ -100,6 +99,12 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return setZones(List.of(zone)); } + public ZoneRegistryMock addZones(ZoneApi... zones) { + List<ZoneApi> allZones = new ArrayList<>(this.zones); + Collections.addAll(allZones, zones); + return setZones(allZones); + } + public ZoneRegistryMock setUpgradePolicy(UpgradePolicy upgradePolicy) { this.upgradePolicy = upgradePolicy; return this; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java index df30b6b57ee..187b8f932cf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java @@ -83,6 +83,13 @@ public class CloudTrialExpirerTest { assertTrue(tester.controller().tenants().get("with-apps").isEmpty()); } + @Test + public void keep_tenants_without_applications_that_are_idle() { + registerTenant("active", "none", Duration.ofDays(364)); + expirer.maintain(); + assertPlan("active", "none"); + } + private void registerTenant(String tenantName, String plan, Duration timeSinceLastLogin) { var name = TenantName.from(tenantName); tester.createTenant(tenantName, Tenant.Type.cloud); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainerTest.java new file mode 100644 index 00000000000..225b7cb1d5e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MeteringMonitorMaintainerTest.java @@ -0,0 +1,51 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClientMock; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.MetricsMock; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +/** + * @author olaa + */ +public class MeteringMonitorMaintainerTest { + + private ControllerTester tester; + private DeploymentTester deploymentTester; + private MetricsMock metrics; + private ResourceDatabaseClientMock database; + private MeteringMonitorMaintainer maintainer; + private final ApplicationId applicationId = ApplicationId.from("foo", "bar", "default"); + private final ZoneId zone = ZoneId.from("prod.aws-us-east-1c"); + + @Before + public void setup() { + tester = new ControllerTester(SystemName.Public); + deploymentTester = new DeploymentTester(tester); + metrics = new MetricsMock(); + database = new ResourceDatabaseClientMock(new PlanRegistryMock()); + maintainer = new MeteringMonitorMaintainer(tester.controller(), Duration.ofMinutes(5), database, metrics); + } + @Test + public void finds_stale_data() { + deploymentTester.newDeploymentContext(applicationId).submit().deploy(); + maintainer.maintain(); + var now = tester.clock().instant().getEpochSecond(); + var lastSnapshot = tester.serviceRegistry().resourceDatabase().getOldestSnapshotTimestamp(Set.of()).getEpochSecond(); + assertEquals(now - lastSnapshot, metrics.getMetric(MeteringMonitorMaintainer.METERING_AGE_METRIC_NAME)); + } + +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index f9441f76a38..a8d39438654 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -9,9 +9,10 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClientMock; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; -import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -38,10 +39,10 @@ import static org.junit.Assert.assertTrue; public class ResourceMeterMaintainerTest { private final ControllerTester tester = new ControllerTester(SystemName.Public); - private final MockMeteringClient snapshotConsumer = new MockMeteringClient(); + private final ResourceDatabaseClientMock resourceClient = new ResourceDatabaseClientMock(new PlanRegistryMock()); private final MetricsMock metrics = new MetricsMock(); private final ResourceMeterMaintainer maintainer = - new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, snapshotConsumer); + new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), metrics, resourceClient); @Test public void updates_deployment_costs() { @@ -64,8 +65,8 @@ public class ResourceMeterMaintainerTest { new ResourceSnapshot(app1, 23, 45, 67, NodeResources.Architecture.getDefault(), Instant.EPOCH, z2), new ResourceSnapshot(app2, 34, 56, 78, NodeResources.Architecture.getDefault(), Instant.EPOCH, z1)); maintainer.updateDeploymentCost(resourceSnapshots); - assertCost.accept(app1, Map.of(z1, 1.40, z2, 2.50)); - assertCost.accept(app2, Map.of(z1, 3.59)); + assertCost.accept(app1, Map.of(z1, 1.72, z2, 3.05)); + assertCost.accept(app2, Map.of(z1, 4.39)); // Remove a region from app1 and add region to app2 resourceSnapshots = List.of( @@ -73,9 +74,9 @@ public class ResourceMeterMaintainerTest { new ResourceSnapshot(app2, 34, 56, 78, NodeResources.Architecture.getDefault(), Instant.EPOCH, z1), new ResourceSnapshot(app2, 45, 67, 89, NodeResources.Architecture.getDefault(), Instant.EPOCH, z2)); maintainer.updateDeploymentCost(resourceSnapshots); - assertCost.accept(app1, Map.of(z2, 2.50)); - assertCost.accept(app2, Map.of(z1, 3.59, z2, 4.68)); - assertEquals(1.4, + assertCost.accept(app1, Map.of(z2, 3.05)); + assertCost.accept(app2, Map.of(z1, 4.39, z2, 5.72)); + assertEquals(1.72, (Double) metrics.getMetric(context -> z1.value().equals(context.get("zoneId")) && app1.tenant().value().equals(context.get("tenant")), @@ -89,7 +90,7 @@ public class ResourceMeterMaintainerTest { long lastRefreshTime = tester.clock().millis(); tester.curator().writeMeteringRefreshTime(lastRefreshTime); maintainer.maintain(); - Collection<ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources(); + Collection<ResourceSnapshot> consumedResources = resourceClient.resourceSnapshots(); // The mocked repository contains two applications, so we should also consume two ResourceSnapshots assertEquals(4, consumedResources.size()); @@ -110,13 +111,13 @@ public class ResourceMeterMaintainerTest { assertEquals(40d, (Double) metrics.getMetric(context -> "tenant2".equals(context.get("tenant")), "metering.vcpu").get(), Double.MIN_VALUE); // Metering is not refreshed - assertFalse(snapshotConsumer.isRefreshed()); + assertFalse(resourceClient.hasRefreshedMaterializedView()); assertEquals(lastRefreshTime, tester.curator().readMeteringRefreshTime()); var millisAdvanced = 3600 * 1000; tester.clock().advance(Duration.ofMillis(millisAdvanced)); maintainer.maintain(); - assertTrue(snapshotConsumer.isRefreshed()); + assertTrue(resourceClient.hasRefreshedMaterializedView()); assertEquals(lastRefreshTime + millisAdvanced, tester.curator().readMeteringRefreshTime()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json index d90e03a1439..3eff2ab781a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json @@ -13,7 +13,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 0.09 + "cost": 0.11 }, "max": { "nodes": 2, @@ -26,7 +26,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 0.35 + "cost": 0.43 }, "current": { "nodes": 2, @@ -39,7 +39,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 0.18 + "cost": 0.22 }, "target": { "nodes": 2, @@ -52,7 +52,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 0.24 + "cost": 0.29 }, "utilization": { "cpu": 0.1, @@ -91,7 +91,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 0.18 + "cost": 0.22 }, "at": 1234, "completion": 2234 diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json index 5268b7b56cb..4d0f1259c07 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json @@ -29,7 +29,7 @@ "commit": "commit1" }, "status": "complete", - "quota": 1.062, + "quota": 1.304, "activity": { }, "metrics": { "queriesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index 481997102d2..a7e2d1913cf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -158,7 +158,6 @@ "jobName": "system-test", "url": "https://some.url:43/instance/default/job/system-test", "environment": "test", - "region": "test.us-east-1", "toRun": [ ], "runs": [ { @@ -348,7 +347,6 @@ "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", - "region": "staging.us-east-3", "toRun": [ { "versions": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index f39aab26d75..696f1ef0ba3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -62,7 +62,6 @@ "jobName": "system-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", "environment": "test", - "region": "test.us-east-1", "toRun": [ ], "runs": [ { @@ -191,7 +190,6 @@ "jobName": "staging-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test", "environment": "staging", - "region": "staging.us-east-3", "toRun": [ ], "runs": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json index 7244cf1493a..b0a8ceeeff0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -29,7 +29,7 @@ "commit": "commit1" }, "status": "complete", - "quota": 1.062, + "quota": 1.304, "activity": { }, "metrics": { "queriesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json index 7244cf1493a..b0a8ceeeff0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-without-shared-endpoints.json @@ -29,7 +29,7 @@ "commit": "commit1" }, "status": "complete", - "quota": 1.062, + "quota": 1.304, "activity": { }, "metrics": { "queriesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 1448b79385b..37bd69d5863 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -54,7 +54,7 @@ "commit": "commit1" }, "status": "complete", - "quota": 1.062, + "quota": 1.304, "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json index 4bd328f605f..d78ab67dcc5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json @@ -66,7 +66,7 @@ "deployTimeEpochMs": 1600000000000, "screwdriverId": "123", "status": "complete", - "quota": 1.062, + "quota": 1.304, "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, @@ -141,7 +141,7 @@ "commit": "commit1" }, "status": "complete", - "quota": 1.062, + "quota": 1.304, "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json index bfbeda9233d..8d76d54458d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json @@ -73,7 +73,7 @@ "deployTimeEpochMs": 1600000000000, "screwdriverId": "123", "status": "complete", - "quota": 1.062, + "quota": 1.304, "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, @@ -148,7 +148,7 @@ "commit": "commit1" }, "status": "complete", - "quota": 1.062, + "quota": 1.304, "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json index e31058d349b..5bf6822baff 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json @@ -392,12 +392,6 @@ "startMillis": 0 } }, - "testReport": { - "summary": { - "success": 1, - "failed": 0 - } - }, "testReports": [ { "summary": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json index 054aaec0fcd..b9bf714d362 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json @@ -72,7 +72,7 @@ "deployTimeEpochMs": 1600000000000, "screwdriverId": "123", "status": "complete", - "quota": 1.062, + "quota": 1.304, "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, @@ -147,7 +147,7 @@ "commit": "commit1" }, "status": "complete", - "quota": 1.062, + "quota": 1.304, "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java index c9e86849ed8..38ebe030e8e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java @@ -254,7 +254,7 @@ public class BillingApiHandlerTest extends ControllerContainerCloudTest { "some-id", "description", new BigDecimal("123.00"), - "some-plan", + "paid", "Smith", addedAt ); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java index 1235dfb33b7..58701732515 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java @@ -76,12 +76,13 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { @Test public void require_admin_for_update_plan() { var request = request("/billing/v2/tenant/" + tenant.value(), Request.Method.PATCH) - .data("{\"plan\": \"pay-as-you-go\"}"); + .data("{\"plan\": \"paid\"}"); var forbidden = request.roles(tenantReader); tester.assertResponse(forbidden, ACCESS_DENIED, 403); var success = request.roles(tenantAdmin); - tester.assertResponse(success, "{\"tenant\":\"tenant1\",\"plan\":\"pay-as-you-go\",\"collection\":\"AUTO\"}"); + tester.assertResponse(success, """ + {"tenant":"tenant1","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"collection":"AUTO"}"""); } @Test @@ -93,7 +94,8 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { tester.assertResponse(forbidden, "{\"error-code\":\"FORBIDDEN\",\"message\":\"Only accountant can change billing method\"}", 403); var success = request.roles(financeAdmin); - tester.assertResponse(success, "{\"tenant\":\"tenant1\",\"plan\":\"trial\",\"collection\":\"INVOICE\"}"); + tester.assertResponse(success, """ + {"tenant":"tenant1","plan":{"id":"trial","name":"Free Trial - for testing purposes"},"collection":"INVOICE"}"""); } @Test @@ -108,7 +110,8 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { tester.assertResponse(listRequest, "{\"invoices\":[{\"id\":\"id-1\",\"from\":\"2020-05-23\",\"to\":\"2020-05-28\",\"total\":\"123.00\",\"status\":\"OPEN\"}]}"); var singleRequest = request("/billing/v2/tenant/" + tenant + "/bill/id-1").roles(tenantReader); - tester.assertResponse(singleRequest, "{\"id\":\"id-1\",\"from\":\"2020-05-23\",\"to\":\"2020-05-28\",\"total\":\"123.00\",\"status\":\"OPEN\",\"statusHistory\":[{\"at\":\"2020-05-23T00:00:00Z\",\"status\":\"OPEN\"}],\"items\":[{\"id\":\"some-id\",\"description\":\"description\",\"amount\":\"123.00\",\"plan\":\"some-plan\",\"planName\":\"Plan with id: some-plan\",\"cpu\":{},\"memory\":{},\"disk\":{}}]}"); + tester.assertResponse(singleRequest, """ + {"id":"id-1","from":"2020-05-23","to":"2020-05-28","total":"123.00","status":"OPEN","statusHistory":[{"at":"2020-05-23T00:00:00Z","status":"OPEN"}],"items":[{"id":"some-id","description":"description","amount":"123.00","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"cpu":{},"memory":{},"disk":{}}]}"""); } @Test @@ -120,7 +123,8 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { "}", 403); var accountantRequest = request("/billing/v2/accountant").roles(Role.hostedAccountant()); - tester.assertResponse(accountantRequest, "{\"tenants\":[{\"tenant\":\"tenant1\",\"plan\":\"trial\",\"collection\":\"AUTO\",\"lastBill\":null,\"unbilled\":\"0.00\"}]}"); + tester.assertResponse(accountantRequest, """ + {"tenants":[{"tenant":"tenant1","plan":{"id":"trial","name":"Free Trial - for testing purposes"},"collection":"AUTO","lastBill":null,"unbilled":"0.00"}]}"""); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json index fe89ef246bb..e9b18a879b9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json @@ -15,8 +15,8 @@ "id": "some-id", "description": "description", "amount": "123.00", - "plan": "some-plan", - "planName": "Plan with id: some-plan" + "plan": "paid", + "planName": "Plan with id: paid" } ] }, @@ -46,8 +46,8 @@ "id": "some-id", "description": "description", "amount": "123.00", - "plan": "some-plan", - "planName": "Plan with id: some-plan" + "plan": "paid", + "planName": "Plan with id: paid" } ] }, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json index 953b946c329..adb319a3642 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json @@ -11,8 +11,8 @@ "id": "some-id", "description": "description", "amount": "123.00", - "plan": "some-plan", - "planName": "Plan with id: some-plan" + "plan": "paid", + "planName": "Plan with id: paid" } ] }, @@ -37,8 +37,8 @@ "id": "some-id", "description": "description", "amount": "123.00", - "plan": "some-plan", - "planName": "Plan with id: some-plan" + "plan": "paid", + "planName": "Plan with id: paid" } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index 94ca4268000..5936c135af9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -161,7 +161,7 @@ public class ControllerApiTest extends ControllerContainerTest { new ResourceSnapshot(applicationId, 12,48,1200, NodeResources.Architecture.arm64, timestamp, zoneId), new ResourceSnapshot(applicationId, 24, 96,2400, NodeResources.Architecture.x86_64, timestamp, zoneId) ); - tester.controller().serviceRegistry().meteringService().consume(snapshots); + tester.controller().serviceRegistry().resourceDatabase().writeResourceSnapshots(snapshots); tester.assertResponse( operatorRequest("http://localhost:8080/controller/v1/metering/tenant/tenantName/month/2020-02", "", Request.Method.GET), new File("metering.json") diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index be28d88abaa..6226e94f7b4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -55,6 +55,9 @@ "name": "JobRunner" }, { + "name": "MeteringMonitorMaintainer" + }, + { "name": "MetricsReporter" }, { diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 1eaaac0f1d3..009cd6d615e 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -172,6 +172,22 @@ function(vespa_use_default_vespa_user) endif() endfunction() +function(vespa_use_default_vespa_group) + if(NOT DEFINED VESPA_GROUP) + if(VESPA_UNPRIVILEGED STREQUAL "no") + set(DEFAULT_VESPA_GROUP "vespa") + if(COMMAND vespa_use_specific_vespa_group) + vespa_use_specific_vespa_group() + endif() + else() + execute_process(COMMAND id -gn ${VESPA_USER} OUTPUT_VARIABLE DEFAULT_VESPA_GROUP) + string(STRIP ${DEFAULT_VESPA_GROUP} DEFAULT_VESPA_GROUP) + endif() + message("-- Setting VESPA_GROUP to ${DEFAULT_VESPA_GROUP}") + set(VESPA_GROUP "${DEFAULT_VESPA_GROUP}" PARENT_SCOPE) + endif() +endfunction() + function(vespa_use_default_build_settings) set(VESPA_DEPS "/opt/vespa-deps") unset(DEFAULT_VESPA_LLVM_VERSION) diff --git a/dist/vespa.spec b/dist/vespa.spec index d278204be2e..de1b5382040 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -78,6 +78,7 @@ BuildRequires: gcc-toolset-11-libatomic-devel %define _devtoolset_enable /opt/rh/gcc-toolset-11/enable %endif BuildRequires: maven +BuildRequires: maven-openjdk17 BuildRequires: pybind11-devel BuildRequires: python3-pytest BuildRequires: python36-devel @@ -145,6 +146,7 @@ BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %if 0%{?el9} BuildRequires: cmake >= 3.20.2 BuildRequires: maven +BuildRequires: maven-openjdk17 BuildRequires: openssl-devel BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.11.0 @@ -162,6 +164,9 @@ BuildRequires: gmock-devel %if 0%{?fedora} BuildRequires: cmake >= 3.9.1 BuildRequires: maven +%if %{?fedora} >= 35 && ! 0%{?amzn2022} +BuildRequires: maven-openjdk17 +%endif BuildRequires: openssl-devel BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.11.0 @@ -687,12 +692,6 @@ exit 0 %postun base if [ $1 -eq 0 ]; then # this is an uninstallation rm -f /etc/profile.d/vespa.sh -%if %{_create_vespa_user} - ! getent passwd %{_vespa_user} >/dev/null || userdel %{_vespa_user} -%endif -%if %{_create_vespa_group} - ! getent group %{_vespa_group} >/dev/null || groupdel %{_vespa_group} -%endif fi # Keep modifications to conf/vespa/default-env.txt across # package uninstall + install. @@ -715,11 +714,14 @@ fi %doc %dir %{_prefix} %{_prefix}/bin +%exclude %{_prefix}/bin/vespa %exclude %{_prefix}/bin/vespa-destination %exclude %{_prefix}/bin/vespa-document-statistics %exclude %{_prefix}/bin/vespa-fbench +%exclude %{_prefix}/bin/vespa-feed-client %exclude %{_prefix}/bin/vespa-feeder %exclude %{_prefix}/bin/vespa-get +%exclude %{_prefix}/bin/vespa-jvm-dumper %exclude %{_prefix}/bin/vespa-logfmt %exclude %{_prefix}/bin/vespa-query-profile-dump-tool %exclude %{_prefix}/bin/vespa-stat @@ -733,8 +735,9 @@ fi %exclude %{_prefix}/conf/configserver-app/components/config-model-fat.jar %exclude %{_prefix}/conf/configserver-app/config-models.xml %dir %{_prefix}/conf/logd +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/conf/telegraf %dir %{_prefix}/conf/vespa -%dir %attr(-,%{_vespa_user},-) %{_prefix}/conf/zookeeper +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/conf/zookeeper %dir %{_prefix}/etc %{_prefix}/etc/systemd %{_prefix}/etc/vespa @@ -742,7 +745,6 @@ fi %{_prefix}/include %dir %{_prefix}/lib %dir %{_prefix}/lib/jars -%{_prefix}/lib/jars/application-model-jar-with-dependencies.jar %{_prefix}/lib/jars/application-preprocessor-jar-with-dependencies.jar %{_prefix}/lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar %{_prefix}/lib/jars/cloud-tenant-cd-jar-with-dependencies.jar @@ -774,24 +776,44 @@ fi %{_prefix}/libexec %exclude %{_prefix}/libexec/vespa_ann_benchmark %exclude %{_prefix}/libexec/vespa/common-env.sh +%exclude %{_prefix}/libexec/vespa/find-pid %exclude %{_prefix}/libexec/vespa/node-admin.sh %exclude %{_prefix}/libexec/vespa/standalone-container.sh %exclude %{_prefix}/libexec/vespa/vespa-curl-wrapper -%dir %attr(1777,-,-) %{_prefix}/logs -%dir %attr(1777,%{_vespa_user},-) %{_prefix}/logs/vespa -%dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/configserver -%dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/node-admin -%dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/search +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/telegraf +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/vespa +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/vespa/access +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/vespa/configserver +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/vespa/node-admin +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/vespa/search %{_prefix}/man %{_prefix}/sbin %{_prefix}/share -%dir %attr(1777,-,-) %{_prefix}/tmp -%dir %attr(1777,%{_vespa_user},-) %{_prefix}/tmp/vespa -%dir %{_prefix}/var -%dir %{_prefix}/var/db -%dir %attr(-,%{_vespa_user},-) %{_prefix}/var/db/vespa -%dir %attr(-,%{_vespa_user},-) %{_prefix}/var/db/vespa/logcontrol -%dir %attr(-,%{_vespa_user},-) %{_prefix}/var/zookeeper +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/tmp +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/tmp/vespa +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/crash +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/db +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/db/vespa +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/db/vespa/config_server +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/db/vespa/config_server/serverdb +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/db/vespa/config_server/serverdb/tenants +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/db/vespa/filedistribution +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/db/vespa/index +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/db/vespa/logcontrol +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/db/vespa/search +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/db/vespa/tmp +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/jdisc_container +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/run +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/vespa +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/vespa/application +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/vespa/bundlecache +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/vespa/bundlecache/configserver +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/vespa/cache +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/vespa/cache/config +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/zookeeper +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/zookeeper/version-2 %config(noreplace) %{_prefix}/conf/logd/logd.cfg %if %{_create_vespa_service} %attr(644,root,root) /usr/lib/systemd/system/vespa.service diff --git a/docproc/pom.xml b/docproc/pom.xml index e7d8d6e3a3e..d656559d5ec 100644 --- a/docproc/pom.xml +++ b/docproc/pom.xml @@ -48,6 +48,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-disc</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>container-messagebus</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/observability/DocprocsStatusExtension.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/observability/DocprocsStatusExtension.java new file mode 100644 index 00000000000..15f6c3a5cd9 --- /dev/null +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/observability/DocprocsStatusExtension.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc.jdisc.observability; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.container.handler.observability.ApplicationStatusHandler; +import com.yahoo.docproc.Call; +import com.yahoo.docproc.impl.DocprocService; +import com.yahoo.docproc.jdisc.DocumentProcessingHandler; +import com.yahoo.jdisc.handler.RequestHandler; + +import java.util.Iterator; +import java.util.Map; + + +/** + * @author bjorncs + */ +public class DocprocsStatusExtension implements ApplicationStatusHandler.Extension { + + @Override + public Map<String, ? extends JsonNode> produceExtraFields(ApplicationStatusHandler statusHandler) { + return Map.of("docprocChains", renderDocprocChains(statusHandler)); + } + + private static JsonNode renderDocprocChains(ApplicationStatusHandler statusHandler) { + ObjectNode ret = statusHandler.jsonMapper().createObjectNode(); + for (RequestHandler h : statusHandler.requestHandlers()) { + if (h instanceof DocumentProcessingHandler) { + ComponentRegistry<DocprocService> registry = ((DocumentProcessingHandler) h).getDocprocServiceRegistry(); + for (DocprocService service : registry.allComponents()) { + ret.set(service.getId().stringValue(), renderCalls(statusHandler, service.getCallStack().iterator())); + } + } + } + return ret; + } + + private static JsonNode renderCalls(ApplicationStatusHandler statusHandler, Iterator<Call> components) { + ArrayNode ret = statusHandler.jsonMapper().createArrayNode(); + while (components.hasNext()) { + Call c = components.next(); + JsonNode jc = ApplicationStatusHandler.renderComponent(c.getDocumentProcessor(), c.getDocumentProcessor().getId()); + ret.add(jc); + } + return ret; + } + +} diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/observability/package-info.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/observability/package-info.java new file mode 100644 index 00000000000..0040acf1fdc --- /dev/null +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/observability/package-info.java @@ -0,0 +1,8 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.docproc.jdisc.observability; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index ab4af5e722e..675ad502630 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -1757,7 +1757,7 @@ public class JsonReaderTestCase { @Test public void tensor_add_update_on_not_fully_specified_cell_throws() { - illegalTensorAddUpdate("Error in 'sparse_tensor': Missing a label for dimension y for tensor(x{},y{})", + illegalTensorAddUpdate("Error in 'sparse_tensor': Missing a label for dimension 'y' for tensor(x{},y{})", "sparse_tensor", "{", " 'cells': [", diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index 89a95c1d62f..a32face5e55 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -134,6 +134,7 @@ FieldPathUpdateTestCase::TearDown() TEST_F(FieldPathUpdateTestCase, testRemoveField) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::things:thangs")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("strfoo") == false); doc->setValue("strfoo", StringFieldValue("cocacola")); EXPECT_EQ(vespalib::string("cocacola"), doc->getValue("strfoo")->getAsString()); @@ -195,6 +196,7 @@ TEST_F(FieldPathUpdateTestCase, testApplyRemoveMultiList2) TEST_F(FieldPathUpdateTestCase, testApplyRemoveEntireListField) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::things:thangs")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("strarray") == false); { ArrayFieldValue strArray(doc->getType().getField("strarray").getDataType()); @@ -213,6 +215,7 @@ TEST_F(FieldPathUpdateTestCase, testApplyRemoveEntireListField) TEST_F(FieldPathUpdateTestCase, testApplyRemoveMultiWset) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::helan:halvan")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("strwset") == false); { WeightedSetFieldValue strWset(doc->getType().getField("strwset").getDataType()); @@ -235,6 +238,7 @@ TEST_F(FieldPathUpdateTestCase, testApplyRemoveMultiWset) TEST_F(FieldPathUpdateTestCase, testApplyAssignSingle) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::drekka:karsk")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("strfoo") == false); // Test assignment of non-existing DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); @@ -252,6 +256,7 @@ TEST_F(FieldPathUpdateTestCase, testApplyAssignSingle) TEST_F(FieldPathUpdateTestCase, testApplyAssignMath) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::bat:man")); + doc->setRepo(*_repo); doc->setValue("num", IntFieldValue(34)); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); @@ -263,6 +268,7 @@ TEST_F(FieldPathUpdateTestCase, testApplyAssignMath) TEST_F(FieldPathUpdateTestCase, testApplyAssignMathByteToZero) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::bat:man")); + doc->setRepo(*_repo); doc->setValue("byteval", ByteFieldValue(3)); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); @@ -275,6 +281,7 @@ TEST_F(FieldPathUpdateTestCase, testApplyAssignMathNotModifiedOnUnderflow) { int low_value = -126; auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::bat:man")); + doc->setRepo(*_repo); doc->setValue("byteval", ByteFieldValue(low_value)); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); @@ -287,6 +294,7 @@ TEST_F(FieldPathUpdateTestCase, testApplyAssignMathNotModifiedOnUnderflow) TEST_F(FieldPathUpdateTestCase, testApplyAssignMathNotModifiedOnOverflow) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::bat:man")); + doc->setRepo(*_repo); doc->setValue("byteval", ByteFieldValue(127)); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); @@ -299,6 +307,7 @@ TEST_F(FieldPathUpdateTestCase, testApplyAssignMathNotModifiedOnOverflow) TEST_F(FieldPathUpdateTestCase, testApplyAssignMathDivZero) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::bat:man")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("num") == false); doc->setValue("num", IntFieldValue(10)); @@ -338,6 +347,7 @@ TEST_F(FieldPathUpdateTestCase, testApplyAssignFieldNotExistingInPath) TEST_F(FieldPathUpdateTestCase, testApplyAssignTargetNotExisting) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::bat:man")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("num") == false); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); @@ -376,6 +386,7 @@ TEST_F(FieldPathUpdateTestCase, testAssignSimpleMapValueWithVariable) TEST_F(FieldPathUpdateTestCase, testApplyAssignMathRemoveIfZero) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::bat:man")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("num") == false); doc->setValue("num", IntFieldValue(34)); EXPECT_TRUE(doc->hasValue("num") == true); @@ -392,6 +403,7 @@ TEST_F(FieldPathUpdateTestCase, testApplyAssignMathRemoveIfZero) TEST_F(FieldPathUpdateTestCase, testApplyAssignMultiList) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::fest:skinnvest")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("strarray") == false); { @@ -421,6 +433,7 @@ TEST_F(FieldPathUpdateTestCase, testApplyAssignMultiList) TEST_F(FieldPathUpdateTestCase, testApplyAssignMultiWset) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::fest:skinnvest")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("strarray") == false); { @@ -451,6 +464,7 @@ TEST_F(FieldPathUpdateTestCase, testApplyAssignMultiWset) TEST_F(FieldPathUpdateTestCase, testAssignWsetRemoveIfZero) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::tronder:bataljon")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("strarray") == false); { @@ -479,7 +493,8 @@ TEST_F(FieldPathUpdateTestCase, testAssignWsetRemoveIfZero) TEST_F(FieldPathUpdateTestCase, testApplyAddMultiList) { - auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::george:costanza")); + auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::george:costanza")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("strarray") == false); auto adds = std::make_unique<ArrayFieldValue>(doc->getType().getField("strarray").getDataType()); @@ -497,7 +512,8 @@ TEST_F(FieldPathUpdateTestCase, testApplyAddMultiList) TEST_F(FieldPathUpdateTestCase, testAddAndAssignList) { - auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::fancy:pants")); + auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::fancy:pants")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("strarray") == false); { @@ -558,11 +574,11 @@ struct Fixture { return sval; } ~Fixture(); - Fixture(const DocumentType &doc_type, const Keys &k); + Fixture(const DocumentTypeRepo &repo, const DocumentType &doc_type, const Keys &k); }; Fixture::~Fixture() = default; -Fixture::Fixture(const DocumentType &doc_type, const Keys &k) +Fixture::Fixture(const DocumentTypeRepo &repo, const DocumentType &doc_type, const Keys &k) : _doc_type(&doc_type), doc(new Document(doc_type, DocumentId("id:ns:" + doc_type.getName() + "::planet:express"))), mfv(getMapType(doc_type)), @@ -570,6 +586,8 @@ Fixture::Fixture(const DocumentType &doc_type, const Keys &k) fv2(getMapType(doc_type).getValueType()), fv3(getMapType(doc_type).getValueType()) { + doc->setRepo(repo); + fv1.setValue("title", StringFieldValue("fry")); fv1.setValue("rating", IntFieldValue(30)); mfv.put(StringFieldValue(k.key1), fv1); @@ -590,7 +608,7 @@ Fixture::Fixture(const DocumentType &doc_type, const Keys &k) TEST_F(FieldPathUpdateTestCase, testAssignMap) { Keys k; - Fixture f(*_foobar_type, k); + Fixture f(*_repo, *_foobar_type, k); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*f.doc->getDataType(), "structmap{" + k.key2 + "}", std::string(), f.fv4())); @@ -606,7 +624,7 @@ TEST_F(FieldPathUpdateTestCase, testAssignMap) TEST_F(FieldPathUpdateTestCase, testAssignMapStruct) { Keys k; - Fixture f(*_foobar_type, k); + Fixture f(*_repo, *_foobar_type, k); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*f.doc->getDataType(), "structmap{" + k.key2 + "}.rating", @@ -623,7 +641,7 @@ TEST_F(FieldPathUpdateTestCase, testAssignMapStruct) TEST_F(FieldPathUpdateTestCase, testAssignMapStructVariable) { Keys k; - Fixture f(*_foobar_type, k); + Fixture f(*_repo, *_foobar_type, k); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*f.doc->getDataType(), "structmap{$x}.rating", @@ -648,6 +666,7 @@ createFry(const DataType & type) { TEST_F(FieldPathUpdateTestCase, testAssignMapNoExist) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::planet:express")); + doc->setRepo(*_repo); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); @@ -665,6 +684,7 @@ TEST_F(FieldPathUpdateTestCase, testAssignMapNoExist) TEST_F(FieldPathUpdateTestCase, testAssignMapNoExistNoCreate) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::planet:express")); + doc->setRepo(*_repo); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); @@ -686,7 +706,7 @@ TEST_F(FieldPathUpdateTestCase, testQuotedStringKey) Keys k; k.key2 = "here is a \"fancy\" 'map' :-} key :-{"; const char field_path[] = "structmap{\"here is a \\\"fancy\\\" 'map' :-} key :-{\"}"; - Fixture f(*_foobar_type, k); + Fixture f(*_repo, *_foobar_type, k); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); docUp.addFieldPathUpdate(std::make_unique<AssignFieldPathUpdate>(*f.doc->getDataType(), field_path, std::string(), f.fv4())); @@ -711,6 +731,7 @@ createTastyCake(const DataType &type) { TEST_F(FieldPathUpdateTestCase, testEqualityComparison) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::foo:zoo")); + doc->setRepo(*_repo); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); { @@ -752,6 +773,7 @@ TEST_F(FieldPathUpdateTestCase, testEqualityComparison) TEST_F(FieldPathUpdateTestCase, testAffectsDocumentBody) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::things:stuff")); + doc->setRepo(*_repo); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); auto fv4 = std::make_unique<StructFieldValue>(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()); @@ -780,6 +802,7 @@ TEST_F(FieldPathUpdateTestCase, testAffectsDocumentBody) TEST_F(FieldPathUpdateTestCase, testIncompatibleDataTypeFails) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::things:stuff")); + doc->setRepo(*_repo); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); @@ -795,6 +818,7 @@ TEST_F(FieldPathUpdateTestCase, testIncompatibleDataTypeFails) TEST_F(FieldPathUpdateTestCase, testSerializeAssign) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::weloveto:serializestuff")); + doc->setRepo(*_repo); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); auto val = std::make_unique<StructFieldValue>(dynamic_cast<const MapDataType&>(*mfv.getDataType()).getValueType()); @@ -813,6 +837,7 @@ TEST_F(FieldPathUpdateTestCase, testSerializeAssign) TEST_F(FieldPathUpdateTestCase, testSerializeAdd) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::george:costanza")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("strarray") == false); auto adds = std::make_unique<ArrayFieldValue>(doc->getType().getField("strarray").getDataType()); @@ -830,6 +855,7 @@ TEST_F(FieldPathUpdateTestCase, testSerializeAdd) TEST_F(FieldPathUpdateTestCase, testSerializeRemove) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::weloveto:serializestuff")); + doc->setRepo(*_repo); MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); DocumentUpdate docUp(*_repo, *_foobar_type, DocumentId("id:ns:foobar::barbar:foofoo")); @@ -842,6 +868,7 @@ TEST_F(FieldPathUpdateTestCase, testSerializeRemove) TEST_F(FieldPathUpdateTestCase, testSerializeAssignMath) { auto doc = std::make_unique<Document>(*_foobar_type, DocumentId("id:ns:foobar::bat:man")); + doc->setRepo(*_repo); EXPECT_TRUE(doc->hasValue("num") == false); doc->setValue("num", IntFieldValue(34)); @@ -933,6 +960,7 @@ TEST_F(FieldPathUpdateTestCase, array_element_update_for_invalid_index_is_ignore TEST_F(FieldPathUpdateTestCase, update_can_have_removes_for_both_existent_and_nonexistent_keys) { DocumentId doc_id("id:ns:foobar::george:costanza"); auto doc = std::make_unique<Document>(*_foobar_type, doc_id); + doc->setRepo(*_repo); auto& map_type = dynamic_cast<const MapDataType&>(doc->getType().getField("structmap").getDataType()); auto& struct_type = map_type.getValueType(); MapFieldValue mfv(map_type); diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 3db001e1732..40d78327ab4 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -812,7 +812,7 @@ DocumenttypesConfig getTensorDocTypesConfig() { const DocumentTypeRepo tensor_doc_repo(getTensorDocTypesConfig()); const FixedTypeRepo tensor_repo(tensor_doc_repo, - *tensor_doc_repo.getDocumentType(doc_type_id)); + *tensor_doc_repo.getDocumentType(tensor_doc_type_id)); const DocumentTypeRepo tensor_doc_repo1(getTensorDocTypesConfig("tensor(dimX{})")); diff --git a/document/src/vespa/document/base/field.h b/document/src/vespa/document/base/field.h index c80733dd5ff..dea7198015a 100644 --- a/document/src/vespa/document/base/field.h +++ b/document/src/vespa/document/base/field.h @@ -56,7 +56,7 @@ public: bool contains(const Set & field) const; size_t size() const { return _fields.size(); } bool empty() const { return _fields.empty(); } - const CPtr * begin() const { return &_fields[0]; } + const CPtr * begin() const { return _fields.data(); } const CPtr * end() const { return begin() + _fields.size(); } static Set emptySet() { return Builder().build(); } private: diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index 529e608533e..7975dd3a327 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -50,8 +50,8 @@ StructFieldValue::getStructType() const { void StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo, uint16_t version, SerializableArray::EntryMap && fm, ByteBuffer buffer) { - _repo = &repo.getDocumentTypeRepo(); - _doc_type = &repo.getDocumentType(); + _repo = repo.getDocumentTypeRepoPtr(); + _doc_type = repo.getDocumentTypePtr(); _version = version; _fields.set(std::move(fm), std::move(buffer)); @@ -117,7 +117,7 @@ StructFieldValue::getField(vespalib::stringref name) const namespace { void -createFV(FieldValue & value, const DocumentTypeRepo & repo, nbostream & stream, const DocumentType & doc_type, uint32_t version) +createFV(FieldValue & value, const DocumentTypeRepo * repo, nbostream & stream, const DocumentType * doc_type, uint32_t version) { FixedTypeRepo frepo(repo, doc_type); try { @@ -142,9 +142,9 @@ StructFieldValue::getFieldValue(const Field& field) const FieldValue::UP value(field.getDataType().createFieldValue()); if ((_repo == nullptr) && (_doc_type != nullptr)) { DocumentTypeRepo tmpRepo(*_doc_type); - createFV(*value, tmpRepo, stream, *_doc_type, _version); + createFV(*value, &tmpRepo, stream, _doc_type, _version); } else { - createFV(*value, *_repo, stream, *_doc_type, _version); + createFV(*value, _repo, stream, _doc_type, _version); } return value; } @@ -172,9 +172,9 @@ StructFieldValue::getFieldValue(const Field& field, FieldValue& value) const nbostream_longlivedbuf stream(buf.c_str(), buf.size()); if ((_repo == nullptr) && (_doc_type != nullptr)) { DocumentTypeRepo tmpRepo(*_doc_type); - createFV(value, tmpRepo, stream, *_doc_type, _version); + createFV(value, &tmpRepo, stream, _doc_type, _version); } else { - createFV(value, *_repo, stream, *_doc_type, _version); + createFV(value, _repo, stream, _doc_type, _version); } return true; } diff --git a/document/src/vespa/document/repo/fixedtyperepo.h b/document/src/vespa/document/repo/fixedtyperepo.h index ca37d893718..f1f9230bfd3 100644 --- a/document/src/vespa/document/repo/fixedtyperepo.h +++ b/document/src/vespa/document/repo/fixedtyperepo.h @@ -17,6 +17,8 @@ public: : _repo(&repo), _doc_type(repo.getDefaultDocType()) {} FixedTypeRepo(const DocumentTypeRepo &repo, const DocumentType &doc_type) noexcept : _repo(&repo), _doc_type(&doc_type) {} + FixedTypeRepo(const DocumentTypeRepo *repo, const DocumentType *doc_type) noexcept + : _repo(repo), _doc_type(doc_type) {} FixedTypeRepo(const DocumentTypeRepo &repo, const vespalib::string &type) noexcept; const DataType *getDataType(int32_t id) const { return _repo->getDataType(*_doc_type, id); } @@ -24,6 +26,8 @@ public: const AnnotationType *getAnnotationType(int32_t id) const { return _repo->getAnnotationType(*_doc_type, id); } const DocumentTypeRepo &getDocumentTypeRepo() const { return *_repo; } const DocumentType &getDocumentType() const noexcept { return *_doc_type; } + const DocumentTypeRepo *getDocumentTypeRepoPtr() const { return _repo; } + const DocumentType *getDocumentTypePtr() const noexcept { return _doc_type; } }; } // namespace document diff --git a/document/src/vespa/document/util/bytebuffer.cpp b/document/src/vespa/document/util/bytebuffer.cpp index 3d38aba6dae..8568ce44c5e 100644 --- a/document/src/vespa/document/util/bytebuffer.cpp +++ b/document/src/vespa/document/util/bytebuffer.cpp @@ -164,7 +164,9 @@ void ByteBuffer::getBytes(void *buffer, uint32_t count) { const char *v = getBufferAtPos(); incPos(count); - memcpy(buffer, v, count); + if (count > 0) { + memcpy(buffer, v, count); + } } } // document diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h index ffbb63e54f1..f55c05f72cd 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h @@ -292,7 +292,6 @@ public: mbus::IRoutingPolicy::UP createPolicy(const mbus::string &name, const mbus::string ¶m) const override; mbus::Blob encode(const vespalib::Version &version, const mbus::Routable &routable) const override; mbus::Routable::UP decode(const vespalib::Version &version, mbus::BlobRef data) const override; - bool requireSequencing() const override { return false; } }; } diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentmessage.cpp index 54c53a938ed..b8d1fbe8015 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentmessage.cpp @@ -2,6 +2,7 @@ #include "removedocumentmessage.h" #include "removedocumentreply.h" #include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/vespalib/util/memory.h> namespace documentapi { @@ -35,7 +36,7 @@ RemoveDocumentMessage::hasSequenceId() const uint64_t RemoveDocumentMessage::getSequenceId() const { - return *reinterpret_cast<const uint64_t*>(_documentId.getGlobalId().get()); + return vespalib::Unaligned<uint64_t>::at(_documentId.getGlobalId().get()).read(); } uint32_t diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 3dca80885f7..960f15eac27 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -82,6 +82,7 @@ vespa_define_module( src/tests/instruction/sparse_full_overlap_join_function src/tests/instruction/sparse_merge_function src/tests/instruction/sparse_no_overlap_join_function + src/tests/instruction/sparse_singledim_lookup src/tests/instruction/sum_max_dot_product_function src/tests/instruction/unpack_bits_function src/tests/instruction/vector_from_doubles_function diff --git a/eval/src/tests/instruction/dense_tensor_peek_function/dense_tensor_peek_function_test.cpp b/eval/src/tests/instruction/dense_tensor_peek_function/dense_tensor_peek_function_test.cpp index 11654a38342..5cd63ce0492 100644 --- a/eval/src/tests/instruction/dense_tensor_peek_function/dense_tensor_peek_function_test.cpp +++ b/eval/src/tests/instruction/dense_tensor_peek_function/dense_tensor_peek_function_test.cpp @@ -56,10 +56,10 @@ TEST("require that tensor peek can be optimized for dense tensors") { TEST_DO(verify("x3y2f{x:(a-1),y:(b)}", 0.0, 1, 0)); } -TEST("require that tensor peek is not optimized for sparse tensor") { +TEST("require that tensor peek is optimized differently for sparse tensor") { TEST_DO(verify("xm{x:1}", 1.0, 0, 1)); - TEST_DO(verify("xm{x:(c)}", 3.0, 0, 1)); - TEST_DO(verify("xm{x:(c+1)}", 0.0, 0, 1)); + TEST_DO(verify("xm{x:(c)}", 3.0, 0, 0)); + TEST_DO(verify("xm{x:(c+1)}", 0.0, 0, 0)); } TEST("require that tensor peek is not optimized for mixed tensor") { @@ -71,10 +71,10 @@ TEST("require that tensor peek is not optimized for mixed tensor") { TEST("require that indexes are truncated when converted to integers") { TEST_DO(verify("x3{x:(a+0.7)}", 2.0, 1, 0)); TEST_DO(verify("x3{x:(a+0.3)}", 2.0, 1, 0)); - TEST_DO(verify("xm{x:(a+0.7)}", 1.0, 0, 1)); - TEST_DO(verify("xm{x:(a+0.3)}", 1.0, 0, 1)); - TEST_DO(verify("xm{x:(-a-0.7)}", 4.0, 0, 1)); - TEST_DO(verify("xm{x:(-a-0.3)}", 4.0, 0, 1)); + TEST_DO(verify("xm{x:(a+0.7)}", 1.0, 0, 0)); + TEST_DO(verify("xm{x:(a+0.3)}", 1.0, 0, 0)); + TEST_DO(verify("xm{x:(-a-0.7)}", 4.0, 0, 0)); + TEST_DO(verify("xm{x:(-a-0.3)}", 4.0, 0, 0)); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/instruction/sparse_singledim_lookup/CMakeLists.txt b/eval/src/tests/instruction/sparse_singledim_lookup/CMakeLists.txt new file mode 100644 index 00000000000..983fd717540 --- /dev/null +++ b/eval/src/tests/instruction/sparse_singledim_lookup/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_sparse_singledim_lookup_test_app TEST + SOURCES + sparse_singledim_lookup_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_sparse_singledim_lookup_test_app COMMAND eval_sparse_singledim_lookup_test_app) diff --git a/eval/src/tests/instruction/sparse_singledim_lookup/sparse_singledim_lookup_test.cpp b/eval/src/tests/instruction/sparse_singledim_lookup/sparse_singledim_lookup_test.cpp new file mode 100644 index 00000000000..7d4e985a5ce --- /dev/null +++ b/eval/src/tests/instruction/sparse_singledim_lookup/sparse_singledim_lookup_test.cpp @@ -0,0 +1,59 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/instruction/sparse_singledim_lookup.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/gen_spec.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +//----------------------------------------------------------------------------- + +struct FunInfo { + using LookFor = SparseSingledimLookup; + void verify(const LookFor &fun) const { + EXPECT_TRUE(fun.result_is_mutable()); + } +}; + +void verify_optimized(const vespalib::string &expr) { + CellTypeSpace type_space(CellTypeUtils::list_types(), 1); + EvalFixture::verify<FunInfo>(expr, {FunInfo()}, type_space); +} + +void verify_not_optimized(const vespalib::string &expr) { + CellTypeSpace just_float({CellType::FLOAT}, 1); + EvalFixture::verify<FunInfo>(expr, {}, just_float); +} + +//----------------------------------------------------------------------------- + +TEST(SparseSingledimLookup, expression_can_be_optimized) { + verify_optimized("x5_1{x:(1+2)}"); +} + +TEST(SparseSingledimLookup, optimized_expression_handles_failed_lookup) { + verify_optimized("x5_1{x:(5+5)}"); + verify_optimized("x5_1{x:(5-10)}"); +} + +TEST(SparseSingledimLookup, verbatim_expression_is_not_optimized) { + verify_not_optimized("x5_1{x:3}"); + verify_not_optimized("x5_1{x:(3)}"); +} + +TEST(SparseSingledimLookup, similar_expressions_are_not_optimized) { + verify_not_optimized("x5{x:(1+2)}"); + verify_not_optimized("x5_1y3{x:(1+2)}"); + verify_not_optimized("x5_1y3_1{x:(1+2)}"); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp index b6258acc9cb..45a4c79d2ed 100644 --- a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp @@ -9,6 +9,7 @@ #include <vespa/eval/instruction/sparse_112_dot_product.h> #include <vespa/eval/instruction/mixed_112_dot_product.h> #include <vespa/eval/instruction/sparse_merge_function.h> +#include <vespa/eval/instruction/sparse_singledim_lookup.h> #include <vespa/eval/instruction/sparse_no_overlap_join_function.h> #include <vespa/eval/instruction/sparse_full_overlap_join_function.h> #include <vespa/eval/instruction/mixed_inner_product_function.h> @@ -100,6 +101,7 @@ const TensorFunction &optimize_for_factory(const ValueBuilderFactory &, const Te child.set(SparseMergeFunction::optimize(child.get(), stash)); child.set(SparseNoOverlapJoinFunction::optimize(child.get(), stash)); child.set(SparseFullOverlapJoinFunction::optimize(child.get(), stash)); + child.set(SparseSingledimLookup::optimize(child.get(), stash)); }); return root.get(); } diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h index c6700b0565a..c5cc99d9137 100644 --- a/eval/src/vespa/eval/eval/tensor_function.h +++ b/eval/src/vespa/eval/eval/tensor_function.h @@ -402,6 +402,7 @@ public: // mapping from dimension name to verbatim label or child index: using Spec = std::map<vespalib::string, LabelOrChildIndex>; Spec make_spec() const; + const TensorFunction ¶m() const { return _param.get(); } const ValueType ¶m_type() const { return _param.get().result_type(); } bool result_is_mutable() const override { return true; } InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const final override; diff --git a/eval/src/vespa/eval/instruction/CMakeLists.txt b/eval/src/vespa/eval/instruction/CMakeLists.txt index f1ec8aa49a9..c4fd0443cbb 100644 --- a/eval/src/vespa/eval/instruction/CMakeLists.txt +++ b/eval/src/vespa/eval/instruction/CMakeLists.txt @@ -42,6 +42,7 @@ vespa_add_library(eval_instruction OBJECT sparse_full_overlap_join_function.cpp sparse_merge_function.cpp sparse_no_overlap_join_function.cpp + sparse_singledim_lookup.cpp sum_max_dot_product_function.cpp unpack_bits_function.cpp vector_from_doubles_function.cpp diff --git a/eval/src/vespa/eval/instruction/sparse_singledim_lookup.cpp b/eval/src/vespa/eval/instruction/sparse_singledim_lookup.cpp new file mode 100644 index 00000000000..4dc453d2a58 --- /dev/null +++ b/eval/src/vespa/eval/instruction/sparse_singledim_lookup.cpp @@ -0,0 +1,84 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "sparse_singledim_lookup.h" +#include <vespa/eval/eval/fast_value.hpp> +#include <vespa/vespalib/util/typify.h> + +namespace vespalib::eval { + +using namespace tensor_function; +using namespace operation; +using namespace instruction; +using Handle = SharedStringRepo::Handle; + +namespace { + +template <typename CT> +double my_sparse_singledim_lookup_fallback(const Value::Index &idx, const CT *cells, string_id key) __attribute__((noinline)); +template <typename CT> +double my_sparse_singledim_lookup_fallback(const Value::Index &idx, const CT *cells, string_id key) { + size_t subspace = 0; + const string_id *key_ref = &key; + auto view = idx.create_view(ConstArrayRef<size_t>{&subspace, 1}); + view->lookup(ConstArrayRef<const string_id *>{&key_ref, 1}); + if (!view->next_result({}, subspace)) { + return 0.0; + } + return cells[subspace]; +} + +template <typename CT> +double my_fast_sparse_singledim_lookup(const FastAddrMap *map, const CT *cells, string_id key) +{ + auto subspace = map->lookup_singledim(key); + return (subspace != FastAddrMap::npos()) ? double(cells[subspace]) : 0.0; +} + +template <typename CT> +void my_sparse_singledim_lookup_op(InterpretedFunction::State &state, uint64_t) { + const auto &idx = state.peek(1).index(); + const CT *cells = state.peek(1).cells().typify<CT>().cbegin(); + int64_t number(state.peek(0).as_double()); + Handle handle = Handle::handle_from_number(number); + double result = __builtin_expect(is_fast(idx), true) + ? my_fast_sparse_singledim_lookup<CT>(&as_fast(idx).map, cells, handle.id()) + : my_sparse_singledim_lookup_fallback<CT>(idx, cells, handle.id()); + state.pop_pop_push(state.stash.create<DoubleValue>(result)); +} + +struct MyGetFun { + template <typename CT> + static auto invoke() { return my_sparse_singledim_lookup_op<CT>; } +}; + +} // namespace <unnamed> + +SparseSingledimLookup::SparseSingledimLookup(const TensorFunction &tensor, + const TensorFunction &expr) + : tensor_function::Op2(ValueType::double_type(), tensor, expr) +{ +} + +InterpretedFunction::Instruction +SparseSingledimLookup::compile_self(const ValueBuilderFactory &, Stash &) const +{ + auto op = typify_invoke<1,TypifyCellType,MyGetFun>(lhs().result_type().cell_type()); + return InterpretedFunction::Instruction(op); +} + +const TensorFunction & +SparseSingledimLookup::optimize(const TensorFunction &expr, Stash &stash) +{ + auto peek = as<Peek>(expr); + if (peek && peek->result_type().is_double() && + peek->param_type().is_sparse() && + (peek->param_type().dimensions().size() == 1) && + (peek->map().size() == 1) && + (std::holds_alternative<TensorFunction::Child>(peek->map().begin()->second))) + { + return stash.create<SparseSingledimLookup>(peek->param(), std::get<TensorFunction::Child>(peek->map().begin()->second).get()); + } + return expr; +} + +} // namespace diff --git a/eval/src/vespa/eval/instruction/sparse_singledim_lookup.h b/eval/src/vespa/eval/instruction/sparse_singledim_lookup.h new file mode 100644 index 00000000000..8c79860361a --- /dev/null +++ b/eval/src/vespa/eval/instruction/sparse_singledim_lookup.h @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::eval { + +/** + * Look up the result of an expression (double->int64_t->label_enum) + * in a sparse tensor with a single dimension, resulting in a double + * result. If lookup keys are kept small [0,10000000> (to avoid label + * enumeration) this is a simple hashtable lookup with numeric keys. + **/ +class SparseSingledimLookup : public tensor_function::Op2 +{ +public: + SparseSingledimLookup(const TensorFunction &tensor, const TensorFunction &expr); + InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + bool result_is_mutable() const override { return true; } + static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); +}; + +} // namespace diff --git a/fastos/src/tests/filetest.cpp b/fastos/src/tests/filetest.cpp index a3470810e2e..d0f8bbfd98b 100644 --- a/fastos/src/tests/filetest.cpp +++ b/fastos/src/tests/filetest.cpp @@ -5,38 +5,7 @@ #include <memory> #include <cassert> #include <sys/mman.h> - -namespace { - -// Create the named file, and write it's filename into it. -// Return true on success -bool createFile(const char* fileName) { - FastOS_StatInfo statInfo; - FastOS_File cf(fileName); - return ( cf.OpenWriteOnly() && - cf.CheckedWrite(fileName, strlen(fileName)) && - cf.Close() && - FastOS_File::Stat(fileName, &statInfo) && - statInfo._isRegular ); -} - -bool createFile(const char* fileName, - const unsigned int size) { - FastOS_File cf(fileName); - bool success = false; - if (cf.OpenWriteOnlyTruncate()) { - auto buf = std::make_unique<char[]>(size); // Could be dangerous.. - if (buf) { - memset(buf.get(), 0, size); // dont write uninitialized bytes since valgrind will complain - if (cf.CheckedWrite(buf.get(), size)) { - success = true; - } - } - } - return success; -} -} // namespace <unnamed> - +#include <filesystem> class FileTest : public BaseTest { @@ -46,176 +15,6 @@ public: const std::string woFilename = "generated/writeonlytest.txt"; const std::string rwFilename = "generated/readwritetest.txt"; - void DirectoryTest() - { - TestHeader ("Directory management (remove & empty) test"); - - const char *dirName = "tmpTestDir"; - char file1[1024]; - char file2[1024]; - char file3[1024]; - char file4[1024]; - char file5[1024]; - char subdir1[512]; - char subdir2[512]; - sprintf(file1, "%s%sfile1", dirName, FastOS_File::GetPathSeparator()); - sprintf(file2, "%s%sfile2", dirName, FastOS_File::GetPathSeparator()); - sprintf(file3, "%s%sfile2", dirName, FastOS_File::GetPathSeparator()); - sprintf(subdir1, "%s%sdir1", dirName, FastOS_File::GetPathSeparator()); - sprintf(subdir2, "%s%sdir2", dirName, FastOS_File::GetPathSeparator()); - sprintf(file4, "%s%sfile4", subdir2, FastOS_File::GetPathSeparator()); - sprintf(file5, "%s%sfile5", subdir2, FastOS_File::GetPathSeparator()); - - FastOS_StatInfo statInfo; - - bool success = false; - - // Don't run at all if the directory already exists - assert(!FastOS_File::Stat(dirName, &statInfo)); - - FastOS_File::MakeDirectory(dirName); - - // Verify that we succeed with an empty directory - FastOS_File::EmptyDirectory(dirName); - success = FastOS_File::Stat(dirName, &statInfo); - Progress(success, "Removing empty directory."); - - // Verify that we can empty a directory with files in it - createFile(file1); - createFile(file2); - createFile(file3); - FastOS_File::EmptyDirectory(dirName); - success = - !FastOS_File::Stat(file1, &statInfo) && - !FastOS_File::Stat(file2, &statInfo) && - !FastOS_File::Stat(file3, &statInfo) && - FastOS_File::Stat(dirName, &statInfo); - Progress(success, "Deleting dir with files in it."); - - // Verify that we can empty a directory with files and directories in it - createFile(file1); - createFile(file2); - createFile(file3); - FastOS_File::MakeDirectory(subdir1); - FastOS_File::MakeDirectory(subdir2); - createFile(file4); - createFile(file5); - FastOS_File::EmptyDirectory(dirName); - success = FastOS_File::Stat(dirName, &statInfo) && - !FastOS_File::Stat(file1, &statInfo) && - !FastOS_File::Stat(file2, &statInfo) && - !FastOS_File::Stat(file3, &statInfo) && - !FastOS_File::Stat(file4, &statInfo) && - !FastOS_File::Stat(file5, &statInfo) && - !FastOS_File::Stat(subdir1, &statInfo) && - !FastOS_File::Stat(subdir2, &statInfo); - Progress(success, "Emptying directory with files and folders in it."); - - // Verify that we don't empty the directory if we find a file to keep - createFile(file1); - createFile(file2); - createFile(file3); - FastOS_File::MakeDirectory(subdir1); - FastOS_File::MakeDirectory(subdir2); - createFile(file4); - createFile(file5); - FastOS_File::EmptyDirectory(dirName, "file1"); - success = FastOS_File::Stat(dirName, &statInfo); - Progress(success, "Emptying dir with keepfile in it."); - // Verify that all but the file to keep are removed - success = FastOS_File::Stat(file1, &statInfo) && - !FastOS_File::Stat(file2, &statInfo) && - !FastOS_File::Stat(file3, &statInfo) && - !FastOS_File::Stat(file4, &statInfo) && - !FastOS_File::Stat(file5, &statInfo) && - !FastOS_File::Stat(subdir1, &statInfo) && - !FastOS_File::Stat(subdir2, &statInfo); - Progress(success, "Looking for keepfile."); - - // Verify that we don't empty the sub-directory if we find a file to keep - createFile(file1); - createFile(file2); - createFile(file3); - FastOS_File::MakeDirectory(subdir1); - FastOS_File::MakeDirectory(subdir2); - createFile(file4); - createFile(file5); - FastOS_File::EmptyDirectory(dirName, "file4"); - success = FastOS_File::Stat(dirName, &statInfo); - Progress(success, "Emptying file with nested keepfile."); - // Verify that all but the file to keep are removed - success = !FastOS_File::Stat(file1, &statInfo) && - !FastOS_File::Stat(file2, &statInfo) && - !FastOS_File::Stat(file3, &statInfo) && - FastOS_File::Stat(file4, &statInfo) && - !FastOS_File::Stat(file5, &statInfo) && - !FastOS_File::Stat(subdir1, &statInfo) && - FastOS_File::Stat(subdir2, &statInfo); - // Progress(success, "Looking for nested keepfile."); // Unsupported for now. - - - FastOS_File::EmptyAndRemoveDirectory(dirName); - - FastOS_File::MakeDirectory(dirName); - - // Verify that we can remove an empty directory - FastOS_File::EmptyAndRemoveDirectory(dirName); - success = !FastOS_File::Stat(dirName, &statInfo); - Progress(success, "Deleting empty directory."); - - // Verify that we can remove a directory with files in it - FastOS_File::MakeDirectory(dirName); - createFile(file1); - createFile(file2); - createFile(file3); - FastOS_File::EmptyAndRemoveDirectory(dirName); - success = !FastOS_File::Stat(dirName, &statInfo); - Progress(success, "Deleting a directory with files in it."); - - // Verify that we can remove a directory with files and directories in it - FastOS_File::MakeDirectory(dirName); - createFile(file1); - createFile(file2); - createFile(file3); - FastOS_File::MakeDirectory(subdir1); - FastOS_File::MakeDirectory(subdir2); - createFile(file4); - createFile(file5); - FastOS_File::EmptyAndRemoveDirectory(dirName); - success = !FastOS_File::Stat(dirName, &statInfo); - Progress(success, "Deleting directory with files and directories in it."); - - } - - void MoveFileTest() { - TestHeader ("Moving files (across volumes too) test"); - - const char *dirName = "tmpTestDir"; - char file1[1024]; - char file2[1024]; - char file3[1024]; - sprintf(file1, "%s%sfile1", dirName, FastOS_File::GetPathSeparator()); - sprintf(file2, "%s%sfile2", dirName, FastOS_File::GetPathSeparator()); - sprintf(file3, "%stmp%sfile3", FastOS_File::GetPathSeparator(), - FastOS_File::GetPathSeparator()); - - FastOS_File::MakeDirectory(dirName); - createFile(file1); - - FastOS_StatInfo statInfo; - // Move file to new name in same dir. - FastOS_File::MoveFile(file1, file2); - Progress(FastOS_File::Stat(file2, &statInfo), "Moving one within a directory."); - - // Move file to /tmp. - FastOS_File::MoveFile(file2, file3); - Progress(FastOS_File::Stat(file3, &statInfo), "Moving to /tmp/"); - - // Clean up - FastOS_File::Delete(file3); - FastOS_File::EmptyAndRemoveDirectory(dirName); - } - void GetCurrentDirTest () { TestHeader ("Get Current Directory Test"); @@ -252,7 +51,7 @@ public: int i; const int bufSize = 1000; - FastOS_File::MakeDirectory("generated"); + std::filesystem::create_directory(std::filesystem::path("generated")); FastOS_File file("generated/memorymaptest"); bool rc = file.OpenReadWrite(); @@ -294,7 +93,7 @@ public: } delete [] buffer; } - FastOS_File::EmptyAndRemoveDirectory("generated"); + std::filesystem::remove_all(std::filesystem::path("generated")); PrintSeparator(); } @@ -305,7 +104,7 @@ public: int i; const int bufSize = 40000; - FastOS_File::MakeDirectory("generated"); + std::filesystem::create_directory(std::filesystem::path("generated")); FastOS_File file("generated/diotest"); bool rc = file.OpenWriteOnly(); @@ -429,7 +228,7 @@ public: delete [] buffer; } - FastOS_File::EmptyAndRemoveDirectory("generated"); + std::filesystem::remove_all(std::filesystem::path("generated")); PrintSeparator(); } @@ -483,7 +282,7 @@ public: void WriteOnlyTest () { TestHeader("Write-Only Test"); - FastOS_File::MakeDirectory("generated"); + std::filesystem::create_directory(std::filesystem::path("generated")); FastOS_File *myFile = new FastOS_File(woFilename.c_str()); @@ -541,14 +340,14 @@ public: Progress(deleteResult, "Delete file '%s'.", woFilename.c_str()); delete(myFile); - FastOS_File::EmptyAndRemoveDirectory("generated"); + std::filesystem::remove_all(std::filesystem::path("generated")); PrintSeparator(); } void ReadWriteTest () { TestHeader("Read/Write Test"); - FastOS_File::MakeDirectory("generated"); + std::filesystem::create_directory(std::filesystem::path("generated")); FastOS_File *myFile = new FastOS_File(rwFilename.c_str()); @@ -637,7 +436,7 @@ public: Progress(deleteResult, "Delete file '%s'.", rwFilename.c_str()); delete(myFile); - FastOS_File::EmptyAndRemoveDirectory("generated"); + std::filesystem::remove_all(std::filesystem::path("generated")); PrintSeparator(); } @@ -715,92 +514,11 @@ public: PrintSeparator(); } - void CopyFileTest () - { - FastOS_StatInfo statInfo; - TestHeader("CopyFile Test"); - const char *dirName = "tmpDir"; - char file1[1024]; - char file2[1024]; - char file3[1024]; - char file4[1024]; - char file5[1024]; - sprintf(file1, "%s%sfile1", dirName, FastOS_File::GetPathSeparator()); - sprintf(file2, "%s%sfile2", dirName, FastOS_File::GetPathSeparator()); - sprintf(file3, "%s%sfile3", dirName, FastOS_File::GetPathSeparator()); - sprintf(file4, "%s%sfile4", dirName, FastOS_File::GetPathSeparator()); - sprintf(file5, "%s%sfile5", dirName, FastOS_File::GetPathSeparator()); - - FastOS_File::EmptyAndRemoveDirectory(dirName); - FastOS_File::MakeDirectory(dirName); - printf("Creating files to copy. Some of them are quite large...\n\n"); - createFile(file1); - createFile(file3, 20*1024*1024); // 20MB file. - createFile(file4, 1024*1024); // 1MB file, i.e. exact size of buffer. - createFile(file5, 1024*1024 + 100); // 1.001MB file - - FastOS_File::Stat(file4, &statInfo); - unsigned int sizeOfFile4 = statInfo._size; - - FastOS_File::Stat(file5, &statInfo); - unsigned int sizeOfFile5 = statInfo._size; - - // Tests start here. - bool copyOK = FastOS_File::CopyFile(file1, file2); - Progress(copyOK, - "File copy from %s to %s.", file1, file2); - - FastOS_File::Delete(file2); - copyOK = FastOS_File::CopyFile(file3, file2); - Progress(copyOK, - "File copy from %s to %s.", file3, file2); - FastOS_File::Stat(file2, &statInfo); - Progress(statInfo._size == 20*1024*1024, - "Size of copied file is 20MB."); - - copyOK = FastOS_File::CopyFile(file3, file3); - Progress(!copyOK, - "File copy onto itself should fail."); - - FastOS_File::Delete(file1); - copyOK = FastOS_File::CopyFile(file1, file2); - Progress(!copyOK, - "File copy of a missing file should fail."); - - copyOK = FastOS_File::CopyFile(file4, file2); - Progress(copyOK, - "Copying a smaller file onto a larger one."); - FastOS_File::Stat(file2, &statInfo); - Progress(statInfo._size == sizeOfFile4, - "Size of copied file should be %u bytes.", sizeOfFile4); - - copyOK = FastOS_File::CopyFile(file4, file1); - Progress(copyOK, - "Copying a file with exact size of buffer."); - FastOS_File::Stat(file1, &statInfo); - Progress(statInfo._size == sizeOfFile4, - "Size of copied file should be %u bytes.", sizeOfFile4); - - copyOK = FastOS_File::CopyFile(file5, file1); - Progress(copyOK, - "Copying a file with size %u bytes.", sizeOfFile5); - FastOS_File::Stat(file1, &statInfo); - Progress(statInfo._size == sizeOfFile5, - "Size of copied file should be %u bytes.", sizeOfFile5); - - - FastOS_File::EmptyAndRemoveDirectory("./tmpDir"); - PrintSeparator(); - } - int Main () override { printf("This test should be run in the 'tests' directory.\n\n"); printf("grep for the string '%s' to detect failures.\n\n", failString); - DirectoryTest(); - MoveFileTest(); - CopyFileTest(); GetCurrentDirTest(); DirectIOTest(); MaxLengthTest(); diff --git a/fastos/src/vespa/fastos/file.cpp b/fastos/src/vespa/fastos/file.cpp index ca109d831a8..fdbacb570b4 100644 --- a/fastos/src/vespa/fastos/file.cpp +++ b/fastos/src/vespa/fastos/file.cpp @@ -207,137 +207,6 @@ FastOS_FileInterface::IsMemoryMapped() const return false; } -bool -FastOS_FileInterface::CopyFile( const char *src, const char *dst ) -{ - FastOS_File s, d; - FastOS_StatInfo statInfo; - bool success = false; - - if ( src != nullptr && - dst != nullptr && - strcmp(src, dst) != 0 && - FastOS_File::Stat( src, &statInfo )) { - - if ( s.OpenReadOnly( src ) && d.OpenWriteOnlyTruncate( dst ) ) { - - unsigned int bufSize = 1024*1024; - int64_t bufSizeBound = statInfo._size; - if (bufSizeBound < 1) - bufSizeBound = 1; - if (bufSizeBound < static_cast<int64_t>(bufSize)) - bufSize = static_cast<unsigned int>(bufSizeBound); - char *tmpBuf = new char[ bufSize ]; - - if ( tmpBuf != nullptr ) { - int64_t copied = 0; - success = true; - do { - unsigned int readBytes = s.Read( tmpBuf, bufSize ); - if (readBytes > 0) { - ssize_t written = d.Write2(tmpBuf, readBytes); - if ( written != readBytes) { - success = false; - } - copied += readBytes; - } else { - // Could not read from src. - success = false; - } - } while (copied < statInfo._size && success); - - delete [] tmpBuf; - } // else out of memory ? - - bool close_ok = s.Close(); - assert(close_ok); - close_ok = d.Close(); - assert(close_ok); - } // else Could not open source or destination file. - } // else Source file does not exist, or input args are invalid. - - return success; -} - - -bool -FastOS_FileInterface::MoveFile(const char* src, const char* dst) -{ - bool rc = FastOS_File::Rename(src, dst); - if (!rc) { - // Try copy and remove. - if (CopyFile(src, dst)) { - rc = FastOS_File::Delete(src); - } - } - return rc; -} - - -void -FastOS_FileInterface::EmptyDirectory( const char *dir, - const char *keepFile /* = nullptr */ ) -{ - FastOS_StatInfo statInfo; - if (!FastOS_File::Stat(dir, &statInfo)) - return; // Fail if the directory does not exist - FastOS_DirectoryScan dirScan( dir ); - - while (dirScan.ReadNext()) { - if (strcmp(dirScan.GetName(), ".") != 0 && - strcmp(dirScan.GetName(), "..") != 0 && - (keepFile == nullptr || strcmp(dirScan.GetName(), keepFile) != 0)) - { - std::string name = dir; - name += GetPathSeparator(); - name += dirScan.GetName(); - if (dirScan.IsDirectory()) { - EmptyAndRemoveDirectory(name.c_str()); - } else { - if ( ! FastOS_File::Delete(name.c_str()) ) { - std::ostringstream os; - os << "Failed deleting file '" << name << "' due to " << getLastErrorString(); - throw std::runtime_error(os.str().c_str()); - } - } - } - } -} - - -void -FastOS_FileInterface::EmptyAndRemoveDirectory(const char *dir) -{ - EmptyDirectory(dir); - FastOS_File::RemoveDirectory(dir); -} - -void -FastOS_FileInterface::MakeDirIfNotPresentOrExit(const char *name) -{ - FastOS_StatInfo statInfo; - - if (FastOS_File::Stat(name, &statInfo)) { - if (statInfo._isDirectory) - return; - - fprintf(stderr, "%s is not a directory\n", name); - std::_Exit(1); - } - - if (statInfo._error != FastOS_StatInfo::FileNotFound) { - std::error_code ec(errno, std::system_category()); - fprintf(stderr, "Could not stat %s: %s\n", name, ec.message().c_str()); - std::_Exit(1); - } - - if (!FastOS_File::MakeDirectory(name)) { - std::error_code ec(errno, std::system_category()); - fprintf(stderr, "Could not mkdir(\"%s\", 0775): %s\n", name, ec.message().c_str()); - std::_Exit(1); - } -} - void FastOS_FileInterface::SetFileName(const char *filename) { diff --git a/fastos/src/vespa/fastos/file.h b/fastos/src/vespa/fastos/file.h index b5e1add3529..34b28b7cc31 100644 --- a/fastos/src/vespa/fastos/file.h +++ b/fastos/src/vespa/fastos/file.h @@ -99,63 +99,6 @@ public: void setFAdviseOptions(int options) { _fAdviseOptions = options; } /** - * Copy a single file. Will overwrite destination if it already exists. - * - * @author Sveinar Rasmussen - * @return success/failure - * @param src a 'const char *' value with the file to copy from - * @param dst a 'const char *' value with the name of the resulting copy - */ - static bool CopyFile( const char *src, const char *dst ); - - /** - * Move a file from src to dst. Has the same semantics as - * FastOS_File::Rename, except that it works across different - * volumes / disks as well (Via copy and remove). - * - * @author Terje Loken - * @return success / failure - * @param src a 'const char *' value with the file to move from - * @param dst a 'const char *' value with the name of the resulting filename - */ - static bool MoveFile( const char *src, const char *dst); - - /** - * Remove a directory, even if it is non-empty. Missing directory does not cause error. - * - * @author Terje Loken - * @throws std::runtime_error if there are errors. - * @param dir a 'const char *' valuem, with the path to the directory we - * want to remove. - */ - static void EmptyAndRemoveDirectory(const char *dir); - - /** - * Empty a directory. Delete all files and directories within the - * dir, with the exception of files matching a specific name - * (optional). The exception does not apply files in - * subdirectories. - * - * @author Terje Loken - * @throws std::runtime_error if there are errors. - * @param dir a 'const char *' value with the directory to empty. - * @param keepFile a 'const char *' value. If supplied, leave files with - * this name alone. - */ - static void EmptyDirectory( const char *dir, const char *keepFile = nullptr); - - /** - * Make a directory (special compatibility version) - * Succeed if the directory already exists. A stat is performed - * to check the directory before attempting to create the - * directory. - * If the procedure fails, an error is printed to stderr and - * the program exits. - * @param name Name of directory - */ - static void MakeDirIfNotPresentOrExit(const char *name); - - /** * Return path separator string. This will yield "/" on UNIX systems. * @return pointer to path separator character string */ diff --git a/fastos/src/vespa/fastos/unix_file.cpp b/fastos/src/vespa/fastos/unix_file.cpp index 39e31c87702..71a9f6e6faa 100644 --- a/fastos/src/vespa/fastos/unix_file.cpp +++ b/fastos/src/vespa/fastos/unix_file.cpp @@ -128,24 +128,6 @@ int FastOS_UNIX_File::GetMaximumPathLength(const char *pathName) return pathconf(pathName, _PC_PATH_MAX); } -bool -FastOS_UNIX_File::MakeDirectory (const char *name) -{ - return (mkdir(name, 0775) == 0); -} - - -void -FastOS_UNIX_File::RemoveDirectory (const char *name) -{ - if ((rmdir(name) != 0) && (ERR_ENOENT != GetLastError())) { - std::ostringstream os; - os << "Remove of directory '" << name << "' failed with error :'" << getLastErrorString() << "'"; - throw std::runtime_error(os.str()); - } -} - - std::string FastOS_UNIX_File::getCurrentDirectory(void) { diff --git a/fastos/src/vespa/fastos/unix_file.h b/fastos/src/vespa/fastos/unix_file.h index 70a8db5036e..368d3c5aca5 100644 --- a/fastos/src/vespa/fastos/unix_file.h +++ b/fastos/src/vespa/fastos/unix_file.h @@ -39,8 +39,6 @@ public: } static bool Stat(const char *filename, FastOS_StatInfo *statInfo); - static bool MakeDirectory(const char *name); - static void RemoveDirectory(const char *name); static std::string getCurrentDirectory(); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 52fcd226a28..7894722ad85 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -122,6 +122,13 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag FEED_NICENESS = defineDoubleFlag( + "feed-niceness", 0.0, + List.of("baldersheim"), "2022-06-24", "2023-01-01", + "How nice feeding shall be", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag SHARED_STRING_REPO_NO_RECLAIM = defineFeatureFlag( "shared-string-repo-no-reclaim", false, List.of("baldersheim"), "2022-06-14", "2023-01-01", @@ -237,7 +244,7 @@ public class Flags { public static final UnboundBooleanFlag FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS = defineFeatureFlag( "fail-deployment-with-invalid-jvm-options", true, - List.of("hmusum"), "2021-12-20", "2022-07-01", + List.of("hmusum"), "2021-12-20", "2022-08-01", "Whether to fail deployments with invalid JVM options in services.xml", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -329,7 +336,7 @@ public class Flags { public static final UnboundStringFlag APPLICATION_FILES_WITH_UNKNOWN_EXTENSION = defineStringFlag( "fail-deployment-for-files-with-unknown-extension", "NOOP", - List.of("hmusum"), "2022-04-27", "2022-06-27", + List.of("hmusum"), "2022-04-27", "2022-07-27", "Whether to log, fail or do nothing for deployments when app has a file with unknown extension (valid values LOG, FAIL, NOOP)", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -357,14 +364,14 @@ public class Flags { public static final UnboundStringFlag FILE_DISTRIBUTION_COMPRESSION_ALGORITHM = defineStringFlag( "file-distribution-compression-algorithm", "gzip", - List.of("hmusum"), "2022-05-24", "2022-06-24", + List.of("hmusum"), "2022-05-24", "2022-07-24", "Which algorithm to use for compressing file references when distributing files. Valid values: none, gzip", "Takes effect immediately", APPLICATION_ID); public static final UnboundBooleanFlag FILE_DISTRIBUTION_COMPRESS_SINGLE_FILES = defineFeatureFlag( "file-distribution-compress-single-files", false, - List.of("hmusum"), "2022-05-24", "2022-06-24", + List.of("hmusum"), "2022-05-24", "2022-07-24", "Whether to compress a file references that is a single file (directories are compressed by default).", "Takes effect immediately", APPLICATION_ID); diff --git a/fnet/src/tests/frt/rpc/invoke.cpp b/fnet/src/tests/frt/rpc/invoke.cpp index 1fbd356b239..e1912985379 100644 --- a/fnet/src/tests/frt/rpc/invoke.cpp +++ b/fnet/src/tests/frt/rpc/invoke.cpp @@ -7,8 +7,10 @@ #include <vespa/fnet/frt/target.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/frt/invoker.h> +#include <vespa/fnet/frt/request_access_filter.h> #include <mutex> #include <condition_variable> +#include <string_view> using vespalib::SocketSpec; using vespalib::BenchmarkTimer; @@ -175,11 +177,25 @@ public: //------------------------------------------------------------- +struct MyAccessFilter : FRT_RequestAccessFilter { + ~MyAccessFilter() override = default; + + constexpr static std::string_view WRONG_KEY = "...mellon!"; + constexpr static std::string_view CORRECT_KEY = "let me in, I have cake"; + + bool allow(FRT_RPCRequest& req) const noexcept override { + const auto& req_param = req.GetParams()->GetValue(0)._string; + const auto magic_key = std::string_view(req_param._str, req_param._len); + return (magic_key == CORRECT_KEY); + } +}; + class TestRPC : public FRT_Invokable { private: - uint32_t _intValue; - RequestLatch _detached_req; + uint32_t _intValue; + RequestLatch _detached_req; + std::atomic<bool> _restricted_method_was_invoked; TestRPC(const TestRPC &); TestRPC &operator=(const TestRPC &); @@ -187,7 +203,8 @@ private: public: TestRPC(FRT_Supervisor *supervisor) : _intValue(0), - _detached_req() + _detached_req(), + _restricted_method_was_invoked(false) { FRT_ReflectionBuilder rb(supervisor); @@ -201,6 +218,9 @@ public: FRT_METHOD(TestRPC::RPC_GetValue), this); rb.DefineMethod("test", "iibb", "i", FRT_METHOD(TestRPC::RPC_Test), this); + rb.DefineMethod("accessRestricted", "s", "", + FRT_METHOD(TestRPC::RPC_AccessRestricted), this); + rb.RequestAccessFilter(std::make_unique<MyAccessFilter>()); } void RPC_Test(FRT_RPCRequest *req) @@ -244,6 +264,16 @@ public: req->GetReturn()->AddInt32(_intValue); } + void RPC_AccessRestricted([[maybe_unused]] FRT_RPCRequest *req) + { + // We'll only get here if the access filter lets us in + _restricted_method_was_invoked.store(true); + } + + bool restricted_method_was_invoked() const noexcept { + return _restricted_method_was_invoked.load(); + } + RequestLatch &detached_req() { return _detached_req; } }; @@ -264,6 +294,7 @@ public: FRT_Target *make_bad_target() { return _client.supervisor().GetTarget("bogus address"); } RequestLatch &detached_req() { return _testRPC.detached_req(); } EchoTest &echo() { return _echoTest; } + const TestRPC& server_instance() const noexcept { return _testRPC; } Fixture() : _client(crypto), @@ -421,6 +452,24 @@ TEST_F("require that parameters can be echoed as return values", Fixture()) { EXPECT_TRUE(req.get().GetParams()->Equals(req.get().GetReturn())); } +TEST_F("request denied by access filter returns PERMISSION_DENIED and does not invoke server method", Fixture()) { + MyReq req("accessRestricted"); + auto key = MyAccessFilter::WRONG_KEY; + req.get().GetParams()->AddString(key.data(), key.size()); + f1.target().InvokeSync(req.borrow(), timeout); + EXPECT_EQUAL(req.get().GetErrorCode(), FRTE_RPC_PERMISSION_DENIED); + EXPECT_FALSE(f1.server_instance().restricted_method_was_invoked()); +} + +TEST_F("request allowed by access filter invokes server method as usual", Fixture()) { + MyReq req("accessRestricted"); + auto key = MyAccessFilter::CORRECT_KEY; + req.get().GetParams()->AddString(key.data(), key.size()); + f1.target().InvokeSync(req.borrow(), timeout); + ASSERT_FALSE(req.get().IsError()); + EXPECT_TRUE(f1.server_instance().restricted_method_was_invoked()); +} + TEST_MAIN() { crypto = my_crypto_engine(); TEST_RUN_ALL(); diff --git a/fnet/src/tests/info/info.cpp b/fnet/src/tests/info/info.cpp index 0d4e0f90a09..4271546e647 100644 --- a/fnet/src/tests/info/info.cpp +++ b/fnet/src/tests/info/info.cpp @@ -80,7 +80,7 @@ TEST("size of important objects") EXPECT_EQUAL(MUTEX_SIZE + sizeof(std::string) + 112u, sizeof(FNET_IOComponent)); EXPECT_EQUAL(32u, sizeof(FNET_Channel)); EXPECT_EQUAL(40u, sizeof(FNET_PacketQueue_NoLock)); - EXPECT_EQUAL(MUTEX_SIZE + sizeof(std::string) + 408u, sizeof(FNET_Connection)); + EXPECT_EQUAL(MUTEX_SIZE + sizeof(std::string) + 416u, sizeof(FNET_Connection)); EXPECT_EQUAL(48u, sizeof(std::condition_variable)); EXPECT_EQUAL(56u, sizeof(FNET_DataBuffer)); EXPECT_EQUAL(8u, sizeof(FNET_Context)); diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp index 2677445e35d..26367c904b2 100644 --- a/fnet/src/vespa/fnet/connection.cpp +++ b/fnet/src/vespa/fnet/connection.cpp @@ -9,6 +9,7 @@ #include "config.h" #include "transport_thread.h" #include "transport.h" +#include <vespa/vespalib/net/connection_auth_context.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/log/log.h> @@ -241,6 +242,8 @@ FNET_Connection::handshake() break; case vespalib::CryptoSocket::HandshakeResult::DONE: { LOG(debug, "Connection(%s): handshake done with peer %s", GetSpec(), GetPeerSpec().c_str()); + _auth_context = _socket->make_auth_context(); + assert(_auth_context); EnableReadEvent(true); EnableWriteEvent(writePendingAfterConnect()); _flags._framed = (_socket->min_read_buffer_size() > 1); @@ -764,3 +767,10 @@ FNET_Connection::GetPeerSpec() const { return vespalib::SocketAddress::peer_address(_socket->get_fd()).spec(); } + +const vespalib::net::ConnectionAuthContext& +FNET_Connection::auth_context() const noexcept +{ + assert(_auth_context); + return *_auth_context; +} diff --git a/fnet/src/vespa/fnet/connection.h b/fnet/src/vespa/fnet/connection.h index 15150ffbb07..4d66f22ce2b 100644 --- a/fnet/src/vespa/fnet/connection.h +++ b/fnet/src/vespa/fnet/connection.h @@ -18,6 +18,8 @@ class FNET_IPacketStreamer; class FNET_IServerAdapter; class FNET_IPacketHandler; +namespace vespalib::net { class ConnectionAuthContext; } + /** * Interface implemented by objects that want to perform connection * cleanup. Use the SetCleanupHandler method to register with a @@ -96,7 +98,7 @@ private: using ResolveHandlerSP = std::shared_ptr<ResolveHandler>; FNET_IPacketStreamer *_streamer; // custom packet streamer FNET_IServerAdapter *_serverAdapter; // only on server side - vespalib::CryptoSocket::UP _socket; // socket for this conn + vespalib::CryptoSocket::UP _socket; // socket for this conn ResolveHandlerSP _resolve_handler; // async resolve callback FNET_Context _context; // connection context std::atomic<State> _state; // connection state. May be polled outside lock @@ -115,6 +117,8 @@ private: FNET_IConnectionCleanupHandler *_cleanup; // cleanup handler + std::unique_ptr<vespalib::net::ConnectionAuthContext> _auth_context; + static std::atomic<uint64_t> _num_connections; // total number of connections @@ -277,7 +281,7 @@ public: /** * Destructor. **/ - ~FNET_Connection(); + ~FNET_Connection() override; /** @@ -504,6 +508,12 @@ public: uint32_t getInputBufferSize() const { return _input.GetBufSize(); } /** + * Returns the connection's auth context. Must only be called _after_ the + * handshake phase has completed. + */ + const vespalib::net::ConnectionAuthContext& auth_context() const noexcept; + + /** * @return the total number of connection objects **/ static uint64_t get_num_connections() { diff --git a/fnet/src/vespa/fnet/frt/CMakeLists.txt b/fnet/src/vespa/fnet/frt/CMakeLists.txt index c7bcbe27041..329c6c8fa57 100644 --- a/fnet/src/vespa/fnet/frt/CMakeLists.txt +++ b/fnet/src/vespa/fnet/frt/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(fnet_frt OBJECT invoker.cpp packets.cpp reflection.cpp + require_capabilities.cpp rpcrequest.cpp supervisor.cpp target.cpp diff --git a/fnet/src/vespa/fnet/frt/error.cpp b/fnet/src/vespa/fnet/frt/error.cpp index 6af9ea39757..fb91924bf35 100644 --- a/fnet/src/vespa/fnet/frt/error.cpp +++ b/fnet/src/vespa/fnet/frt/error.cpp @@ -12,19 +12,20 @@ FRT_GetErrorCodeName(uint32_t errorCode) errorCode <= FRTE_RPC_LAST) { switch (errorCode) { - case FRTE_RPC_GENERAL_ERROR: return "FRTE_RPC_GENERAL_ERROR"; - case FRTE_RPC_NOT_IMPLEMENTED: return "FRTE_RPC_NOT_IMPLEMENTED"; - case FRTE_RPC_ABORT: return "FRTE_RPC_ABORT"; - case FRTE_RPC_TIMEOUT: return "FRTE_RPC_TIMEOUT"; - case FRTE_RPC_CONNECTION: return "FRTE_RPC_CONNECTION"; - case FRTE_RPC_BAD_REQUEST: return "FRTE_RPC_BAD_REQUEST"; - case FRTE_RPC_NO_SUCH_METHOD: return "FRTE_RPC_NO_SUCH_METHOD"; - case FRTE_RPC_WRONG_PARAMS: return "FRTE_RPC_WRONG_PARAMS"; - case FRTE_RPC_OVERLOAD: return "FRTE_RPC_OVERLOAD"; - case FRTE_RPC_WRONG_RETURN: return "FRTE_RPC_WRONG_RETURN"; - case FRTE_RPC_BAD_REPLY: return "FRTE_RPC_BAD_REPLY"; - case FRTE_RPC_METHOD_FAILED: return "FRTE_RPC_METHOD_FAILED"; - default: return "[UNKNOWN RPC ERROR]"; + case FRTE_RPC_GENERAL_ERROR: return "FRTE_RPC_GENERAL_ERROR"; + case FRTE_RPC_NOT_IMPLEMENTED: return "FRTE_RPC_NOT_IMPLEMENTED"; + case FRTE_RPC_ABORT: return "FRTE_RPC_ABORT"; + case FRTE_RPC_TIMEOUT: return "FRTE_RPC_TIMEOUT"; + case FRTE_RPC_CONNECTION: return "FRTE_RPC_CONNECTION"; + case FRTE_RPC_BAD_REQUEST: return "FRTE_RPC_BAD_REQUEST"; + case FRTE_RPC_NO_SUCH_METHOD: return "FRTE_RPC_NO_SUCH_METHOD"; + case FRTE_RPC_WRONG_PARAMS: return "FRTE_RPC_WRONG_PARAMS"; + case FRTE_RPC_OVERLOAD: return "FRTE_RPC_OVERLOAD"; + case FRTE_RPC_WRONG_RETURN: return "FRTE_RPC_WRONG_RETURN"; + case FRTE_RPC_BAD_REPLY: return "FRTE_RPC_BAD_REPLY"; + case FRTE_RPC_METHOD_FAILED: return "FRTE_RPC_METHOD_FAILED"; + case FRTE_RPC_PERMISSION_DENIED: return "FRTE_RPC_PERMISSION_DENIED"; + default: return "[UNKNOWN RPC ERROR]"; } } return "[UNKNOWN ERROR]"; @@ -41,19 +42,20 @@ FRT_GetDefaultErrorMessage(uint32_t errorCode) errorCode <= FRTE_RPC_LAST) { switch (errorCode) { - case FRTE_RPC_GENERAL_ERROR: return "(RPC) General error"; - case FRTE_RPC_NOT_IMPLEMENTED: return "(RPC) Not implemented"; - case FRTE_RPC_ABORT: return "(RPC) Invocation aborted"; - case FRTE_RPC_TIMEOUT: return "(RPC) Invocation timed out"; - case FRTE_RPC_CONNECTION: return "(RPC) Connection error"; - case FRTE_RPC_BAD_REQUEST: return "(RPC) Bad request packet"; - case FRTE_RPC_NO_SUCH_METHOD: return "(RPC) No such method"; - case FRTE_RPC_WRONG_PARAMS: return "(RPC) Illegal parameters"; - case FRTE_RPC_OVERLOAD: return "(RPC) Request dropped due to server overload"; - case FRTE_RPC_WRONG_RETURN: return "(RPC) Illegal return values"; - case FRTE_RPC_BAD_REPLY: return "(RPC) Bad reply packet"; - case FRTE_RPC_METHOD_FAILED: return "(RPC) Method failed"; - default: return "[UNKNOWN RPC ERROR]"; + case FRTE_RPC_GENERAL_ERROR: return "(RPC) General error"; + case FRTE_RPC_NOT_IMPLEMENTED: return "(RPC) Not implemented"; + case FRTE_RPC_ABORT: return "(RPC) Invocation aborted"; + case FRTE_RPC_TIMEOUT: return "(RPC) Invocation timed out"; + case FRTE_RPC_CONNECTION: return "(RPC) Connection error"; + case FRTE_RPC_BAD_REQUEST: return "(RPC) Bad request packet"; + case FRTE_RPC_NO_SUCH_METHOD: return "(RPC) No such method"; + case FRTE_RPC_WRONG_PARAMS: return "(RPC) Illegal parameters"; + case FRTE_RPC_OVERLOAD: return "(RPC) Request dropped due to server overload"; + case FRTE_RPC_WRONG_RETURN: return "(RPC) Illegal return values"; + case FRTE_RPC_BAD_REPLY: return "(RPC) Bad reply packet"; + case FRTE_RPC_METHOD_FAILED: return "(RPC) Method failed"; + case FRTE_RPC_PERMISSION_DENIED: return "(RPC) Permission denied"; + default: return "[UNKNOWN RPC ERROR]"; } } return "[UNKNOWN ERROR]"; diff --git a/fnet/src/vespa/fnet/frt/error.h b/fnet/src/vespa/fnet/frt/error.h index c5acfb744f6..7b3cdc7320b 100644 --- a/fnet/src/vespa/fnet/frt/error.h +++ b/fnet/src/vespa/fnet/frt/error.h @@ -4,21 +4,22 @@ #include <cstdint> enum { - FRTE_NO_ERROR = 0, - FRTE_RPC_FIRST = 100, - FRTE_RPC_GENERAL_ERROR = 100, - FRTE_RPC_NOT_IMPLEMENTED = 101, - FRTE_RPC_ABORT = 102, - FRTE_RPC_TIMEOUT = 103, - FRTE_RPC_CONNECTION = 104, - FRTE_RPC_BAD_REQUEST = 105, - FRTE_RPC_NO_SUCH_METHOD = 106, - FRTE_RPC_WRONG_PARAMS = 107, - FRTE_RPC_OVERLOAD = 108, - FRTE_RPC_WRONG_RETURN = 109, - FRTE_RPC_BAD_REPLY = 110, - FRTE_RPC_METHOD_FAILED = 111, - FRTE_RPC_LAST = 199 + FRTE_NO_ERROR = 0, + FRTE_RPC_FIRST = 100, + FRTE_RPC_GENERAL_ERROR = 100, + FRTE_RPC_NOT_IMPLEMENTED = 101, + FRTE_RPC_ABORT = 102, + FRTE_RPC_TIMEOUT = 103, + FRTE_RPC_CONNECTION = 104, + FRTE_RPC_BAD_REQUEST = 105, + FRTE_RPC_NO_SUCH_METHOD = 106, + FRTE_RPC_WRONG_PARAMS = 107, + FRTE_RPC_OVERLOAD = 108, + FRTE_RPC_WRONG_RETURN = 109, + FRTE_RPC_BAD_REPLY = 110, + FRTE_RPC_METHOD_FAILED = 111, + FRTE_RPC_PERMISSION_DENIED = 112, + FRTE_RPC_LAST = 199 }; const char *FRT_GetErrorCodeName(uint32_t errorCode); diff --git a/fnet/src/vespa/fnet/frt/invoker.cpp b/fnet/src/vespa/fnet/frt/invoker.cpp index 85eae6cb41a..d4b35720a8d 100644 --- a/fnet/src/vespa/fnet/frt/invoker.cpp +++ b/fnet/src/vespa/fnet/frt/invoker.cpp @@ -52,6 +52,7 @@ FRT_RPCInvoker::FRT_RPCInvoker(FRT_Supervisor *supervisor, std::string methodName(_req->GetMethodName(), _req->GetMethodNameLen()); LOG(debug, "invoke(server) init: '%s'", methodName.c_str()); } + req->SetReturnHandler(this); // RPC req -> FNET_Connection link is via this ptr; set prior to access filter invocation. if (_method == nullptr) { if (!req->IsError()) { // may be BAD_REQUEST req->SetError(FRTE_RPC_NO_SUCH_METHOD); @@ -60,8 +61,11 @@ FRT_RPCInvoker::FRT_RPCInvoker(FRT_Supervisor *supervisor, req->GetParamSpec())) { req->SetError(FRTE_RPC_WRONG_PARAMS); + } else if (_method->GetRequestAccessFilter() && + !_method->GetRequestAccessFilter()->allow(*req)) + { + req->SetError(FRTE_RPC_PERMISSION_DENIED); } - req->SetReturnHandler(this); } bool FRT_RPCInvoker::Invoke() diff --git a/fnet/src/vespa/fnet/frt/reflection.cpp b/fnet/src/vespa/fnet/frt/reflection.cpp index 211e681df94..af7fa069eb9 100644 --- a/fnet/src/vespa/fnet/frt/reflection.cpp +++ b/fnet/src/vespa/fnet/frt/reflection.cpp @@ -14,7 +14,8 @@ FRT_Method::FRT_Method(const char * name, const char * paramSpec, const char * r _returnSpec(returnSpec), _method(method), _handler(handler), - _doc() + _doc(), + _access_filter() { } @@ -124,6 +125,7 @@ FRT_ReflectionBuilder::Flush() } _method->SetDocumentation(_values); + _method->SetRequestAccessFilter(std::move(_access_filter)); // May be nullptr _method = nullptr; _req->Reset(); } @@ -142,7 +144,8 @@ FRT_ReflectionBuilder::FRT_ReflectionBuilder(FRT_Supervisor *supervisor) _arg_name(nullptr), _arg_desc(nullptr), _ret_name(nullptr), - _ret_desc(nullptr) + _ret_desc(nullptr), + _access_filter() { } @@ -183,6 +186,7 @@ FRT_ReflectionBuilder::DefineMethod(const char *name, _arg_desc = _values->AddStringArray(_argCnt); _ret_name = _values->AddStringArray(_retCnt); _ret_desc = _values->AddStringArray(_retCnt); + _access_filter.reset(); } @@ -224,3 +228,12 @@ FRT_ReflectionBuilder::ReturnDesc(const char *name, const char *desc) _values->SetString(&_ret_desc[_curRet], desc); _curRet++; } + +void +FRT_ReflectionBuilder::RequestAccessFilter(std::unique_ptr<FRT_RequestAccessFilter> access_filter) +{ + if (_method == nullptr) { + return; + } + _access_filter = std::move(access_filter); +} diff --git a/fnet/src/vespa/fnet/frt/reflection.h b/fnet/src/vespa/fnet/frt/reflection.h index 6267cafeeb1..3f833d053f1 100644 --- a/fnet/src/vespa/fnet/frt/reflection.h +++ b/fnet/src/vespa/fnet/frt/reflection.h @@ -3,6 +3,8 @@ #pragma once #include "invokable.h" +#include "request_access_filter.h" +#include <memory> #include <string> #include <vector> @@ -23,6 +25,7 @@ private: FRT_METHOD_PT _method; // method pointer FRT_Invokable *_handler; // method handler std::vector<char> _doc; // method documentation + std::unique_ptr<FRT_RequestAccessFilter> _access_filter; // (optional) access filter public: FRT_Method(const FRT_Method &) = delete; @@ -41,6 +44,10 @@ public: const char *GetReturnSpec() { return _returnSpec.c_str(); } FRT_METHOD_PT GetMethod() { return _method; } FRT_Invokable *GetHandler() { return _handler; } + const FRT_RequestAccessFilter* GetRequestAccessFilter() const noexcept { return _access_filter.get(); } + void SetRequestAccessFilter(std::unique_ptr<FRT_RequestAccessFilter> access_filter) noexcept { + _access_filter = std::move(access_filter); + } void SetDocumentation(FRT_Values *values); void GetDocumentation(FRT_Values *values); }; @@ -104,6 +111,7 @@ private: FRT_StringValue *_arg_desc; FRT_StringValue *_ret_name; FRT_StringValue *_ret_desc; + std::unique_ptr<FRT_RequestAccessFilter> _access_filter; FRT_ReflectionBuilder(const FRT_ReflectionBuilder &); FRT_ReflectionBuilder &operator=(const FRT_ReflectionBuilder &); @@ -122,5 +130,6 @@ public: void MethodDesc(const char *desc); void ParamDesc(const char *name, const char *desc); void ReturnDesc(const char *name, const char *desc); + void RequestAccessFilter(std::unique_ptr<FRT_RequestAccessFilter> access_filter); }; diff --git a/fnet/src/vespa/fnet/frt/request_access_filter.h b/fnet/src/vespa/fnet/frt/request_access_filter.h new file mode 100644 index 00000000000..a02dca646f3 --- /dev/null +++ b/fnet/src/vespa/fnet/frt/request_access_filter.h @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +class FRT_RPCRequest; + +/** + * An RPC request access filter will, if provided during method registration, be + * invoked _prior_ to any RPC handler callback invocation for that method. It allows + * for implementing method-specific authorization handling, logging etc. + * + * Must be thread safe. + */ +class FRT_RequestAccessFilter { +public: + virtual ~FRT_RequestAccessFilter() = default; + + /** + * Iff true is returned, the request is allowed through and the RPC callback + * will be invoked as usual. If false, the request is immediately failed back + * to the caller with an error code. + */ + [[nodiscard]] virtual bool allow(FRT_RPCRequest&) const noexcept = 0; +}; diff --git a/fnet/src/vespa/fnet/frt/require_capabilities.cpp b/fnet/src/vespa/fnet/frt/require_capabilities.cpp new file mode 100644 index 00000000000..c74e9ad648a --- /dev/null +++ b/fnet/src/vespa/fnet/frt/require_capabilities.cpp @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "require_capabilities.h" +#include "rpcrequest.h" +#include <vespa/fnet/connection.h> +#include <vespa/vespalib/net/connection_auth_context.h> + +bool +FRT_RequireCapabilities::allow(FRT_RPCRequest& req) const noexcept +{ + const auto& auth_ctx = req.GetConnection()->auth_context(); + return auth_ctx.capabilities().contains_all(_required_capabilities); +} diff --git a/fnet/src/vespa/fnet/frt/require_capabilities.h b/fnet/src/vespa/fnet/frt/require_capabilities.h new file mode 100644 index 00000000000..7c80484783d --- /dev/null +++ b/fnet/src/vespa/fnet/frt/require_capabilities.h @@ -0,0 +1,21 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "request_access_filter.h" +#include <vespa/vespalib/net/tls/capability_set.h> + +/** + * An RPC access filter which verifies that a request is associated with an auth + * context that contains, at minimum, a given set of capabilities. If one or more + * required capabilities are missing, the request is denied. + */ +class FRT_RequireCapabilities final : public FRT_RequestAccessFilter { + vespalib::net::tls::CapabilitySet _required_capabilities; +public: + explicit constexpr FRT_RequireCapabilities(vespalib::net::tls::CapabilitySet required_capabilities) noexcept + : _required_capabilities(required_capabilities) + { + } + + bool allow(FRT_RPCRequest& req) const noexcept override; +}; diff --git a/fsa/src/vespa/fsa/automaton.cpp b/fsa/src/vespa/fsa/automaton.cpp index 91592078c98..57bcfb180a8 100644 --- a/fsa/src/vespa/fsa/automaton.cpp +++ b/fsa/src/vespa/fsa/automaton.cpp @@ -11,7 +11,7 @@ #include "fsa.h" #include "automaton.h" #include "checksum.h" - +#include "unaligned.h" namespace fsa { @@ -354,7 +354,7 @@ void Automaton::PackedAutomaton::finalize() uint32_t j=lastsize; bool fixedsize = true; while(i<_blob_used){ - currsize = *((uint32_t*)(void *)(_blob+i)); + currsize = Unaligned<uint32_t>::at(_blob+i); if(currsize!=lastsize){ fixedsize = false; break; @@ -597,14 +597,14 @@ bool Automaton::PackedAutomaton::getFSA(FSA::Descriptor &d) d._version = FSA::VER; d._serial = 0; - d._state = _packed_idx; + d._state = Unaligned<state_t>::ptr(_packed_idx); d._symbol = _symbol; d._size = size; d._data = _blob; d._data_size = _blob_used; d._data_type = _blob_type; d._fixed_data_size = _fixed_blob_size; - d._perf_hash = _perf_hash; + d._perf_hash = Unaligned<hash_t>::ptr(_perf_hash); d._start = _start_state; _symbol = NULL; diff --git a/fsa/src/vespa/fsa/fsa.cpp b/fsa/src/vespa/fsa/fsa.cpp index 50fd8bff85d..4abc6f979d8 100644 --- a/fsa/src/vespa/fsa/fsa.cpp +++ b/fsa/src/vespa/fsa/fsa.cpp @@ -227,7 +227,7 @@ bool FSA::read(const char *file, FileAccessMethod fam) checksum += Checksum::compute(_symbol,_size*sizeof(symbol_t)); if(_mmap_addr==NULL){ - _state = (state_t*)malloc(_size*sizeof(state_t)); + _state = Unaligned<state_t>::ptr(malloc(_size*sizeof(state_t))); r=::read(fd,_state,_size*sizeof(state_t)); if(r!=_size*sizeof(state_t)){ ::close(fd); @@ -236,8 +236,8 @@ bool FSA::read(const char *file, FileAccessMethod fam) } } else { - _state = (state_t*)(void *)((uint8_t*)_mmap_addr + sizeof(header) + - _size*sizeof(symbol_t)); + _state = Unaligned<state_t>::ptr((uint8_t*)_mmap_addr + sizeof(header) + + _size*sizeof(symbol_t)); } checksum += Checksum::compute(_state,_size*sizeof(state_t)); @@ -259,7 +259,7 @@ bool FSA::read(const char *file, FileAccessMethod fam) if(header._has_perfect_hash){ if(_mmap_addr==NULL){ - _perf_hash = (hash_t*)malloc(_size*sizeof(hash_t)); + _perf_hash = Unaligned<hash_t>::ptr(malloc(_size*sizeof(hash_t))); r=::read(fd,_perf_hash,_size*sizeof(hash_t)); if(r!=_size*sizeof(hash_t)){ ::close(fd); @@ -268,10 +268,10 @@ bool FSA::read(const char *file, FileAccessMethod fam) } } else { - _perf_hash = (hash_t*)(void *)((uint8_t*)_mmap_addr + sizeof(header) + - _size*sizeof(symbol_t) + - _size*sizeof(state_t) + - _data_size); + _perf_hash = Unaligned<hash_t>::ptr((uint8_t*)_mmap_addr + sizeof(header) + + _size*sizeof(symbol_t) + + _size*sizeof(state_t) + + _data_size); } checksum += Checksum::compute(_perf_hash,_size*sizeof(hash_t)); _has_perfect_hash = true; diff --git a/fsa/src/vespa/fsa/fsa.h b/fsa/src/vespa/fsa/fsa.h index 915aeb17a17..f3703b398ba 100644 --- a/fsa/src/vespa/fsa/fsa.h +++ b/fsa/src/vespa/fsa/fsa.h @@ -15,6 +15,7 @@ #include <inttypes.h> #include "file.h" // for FileAccessMethod +#include "unaligned.h" namespace fsa { @@ -614,10 +615,10 @@ public: return (uint32_t)((const uint8_t*)da)[0]; case 2: case 3: - return (uint32_t)((const uint16_t*)(const void *)da)[0]; + return (uint32_t)Unaligned<uint16_t>::at(da).read(); case 4: default: - return ((const uint32_t*)(const void *) da)[0]; + return Unaligned<uint32_t>::at(da).read(); } } @@ -2019,14 +2020,14 @@ public: struct Descriptor { uint32_t _version; uint32_t _serial; - state_t *_state; + Unaligned<state_t> *_state; symbol_t *_symbol; uint32_t _size; data_t *_data; uint32_t _data_size; uint32_t _data_type; uint32_t _fixed_data_size; - hash_t *_perf_hash; + Unaligned<hash_t> *_perf_hash; uint32_t _start; }; @@ -2040,7 +2041,7 @@ private: uint32_t _version; /**< Version of fsalib used to build this fsa. */ uint32_t _serial; /**< Serial number of this fsa. */ - state_t *_state; /**< State table for transitions. */ + Unaligned<state_t> *_state; /**< State table for transitions. */ symbol_t *_symbol; /**< Symbol table for transitions. */ uint32_t _size; /**< Size (number of cells). */ @@ -2050,7 +2051,7 @@ private: uint32_t _fixed_data_size; /**< Size of data items if fixed. */ bool _has_perfect_hash; /**< Indicator of perfect hash present. */ - hash_t *_perf_hash; /**< Perfect hash table, if present. */ + Unaligned<hash_t> *_perf_hash; /**< Perfect hash table, if present. */ state_t _start; /**< Index of start state. */ @@ -2205,7 +2206,7 @@ public: if(_data_type==DATA_FIXED) return _fixed_data_size; else - return (int)(*((uint32_t*)(void *)(_data+_state[fs+FINAL_SYMBOL]))); + return (int)Unaligned<uint32_t>::at(_data+_state[fs+FINAL_SYMBOL]).read(); } return -1; } diff --git a/fsa/src/vespa/fsa/unaligned.h b/fsa/src/vespa/fsa/unaligned.h new file mode 100644 index 00000000000..ff645194e4f --- /dev/null +++ b/fsa/src/vespa/fsa/unaligned.h @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstring> + +namespace fsa { + +template <typename T> +class Unaligned { +private: + char _data[sizeof(T)]; + +public: + Unaligned() = delete; + Unaligned(const Unaligned &) = delete; + Unaligned(Unaligned &&) = delete; + + Unaligned &operator=(const Unaligned &) = default; + Unaligned &operator=(Unaligned &&) = default; + + static_assert(std::is_trivial_v<T>); + static_assert(alignof(T) > 1, "value is always aligned"); + + constexpr static Unaligned &at(void *p) noexcept { + return *reinterpret_cast<Unaligned*>(p); + } + constexpr static const Unaligned &at(const void *p) noexcept { + return *reinterpret_cast<const Unaligned*>(p); + } + + constexpr static Unaligned *ptr(void *p) noexcept { + return reinterpret_cast<Unaligned*>(p); + } + constexpr static const Unaligned *ptr(const void *p) noexcept { + return reinterpret_cast<const Unaligned*>(p); + } + + T read() const noexcept { + T value; + static_assert(sizeof(_data) == sizeof(value)); + memcpy(&value, _data, sizeof(value)); + return value; + } + void write(const T &value) noexcept { + static_assert(sizeof(_data) == sizeof(value)); + memcpy(_data, &value, sizeof(value)); + } + operator T () const noexcept { return read(); } + Unaligned &operator=(const T &value) noexcept { + write(value); + return *this; + } +}; + +} diff --git a/functions.cmake b/functions.cmake index 3721d26d0df..98662d844fc 100644 --- a/functions.cmake +++ b/functions.cmake @@ -749,10 +749,6 @@ function(vespa_detect_build_platform) endif() endfunction() -function(vespa_install_empty_tmp_dir TARGET) -install(DIRECTORY DESTINATION ${TARGET} DIRECTORY_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_WRITE GROUP_EXECUTE WORLD_READ WORLD_WRITE WORLD_EXECUTE SETGID) -endfunction() - function(vespa_suppress_warnings_for_protobuf_sources) cmake_parse_arguments( ARG diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 39b555ea105..8bc98cf0733 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -36,7 +36,8 @@ <target_jdk_version>17</target_jdk_version> <maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version> <maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version> - <junit.version>5.8.1</junit.version> <!-- NOTE: this must be in sync with junit version specified in 'tenant-cd-api' --> + <!-- NOTE: this must not be overriden by users, and must be in sync with junit version specified in 'tenant-cd-api' --> + <vespa.junit.version>5.8.1</vespa.junit.version> <test.categories>!integration</test.categories> <!-- To allow specialized base pom to include additional "test provided" dependencies --> @@ -94,13 +95,13 @@ <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-engine</artifactId> - <version>${junit.version}</version> + <version>${vespa.junit.version}</version> </dependency> <dependency> <groupId>org.junit.vintage</groupId> <artifactId>junit-vintage-engine</artifactId> - <version>${junit.version}</version> + <version>${vespa.junit.version}</version> </dependency> </dependencies> </dependencyManagement> @@ -235,8 +236,8 @@ <exclude>org.junit.jupiter:junit-jupiter-api:*:jar:compile</exclude> <exclude>org.junit.jupiter:junit-jupiter-api:*:jar:provided</exclude> <exclude>org.junit.jupiter:junit-jupiter-api:*:jar:runtime</exclude> - <exclude>org.junit.jupiter:junit-jupiter-api:(,${junit.version}):jar:*</exclude> - <exclude>org.junit.jupiter:junit-jupiter-api:(${junit.version},):jar:*</exclude> + <exclude>org.junit.jupiter:junit-jupiter-api:(,${vespa.junit.version}):jar:*</exclude> + <exclude>org.junit.jupiter:junit-jupiter-api:(${vespa.junit.version},):jar:*</exclude> </excludes> </bannedDependencies> </rules> diff --git a/hosted-zone-api/abi-spec.json b/hosted-zone-api/abi-spec.json index 826f731b4e0..0d9a6409759 100644 --- a/hosted-zone-api/abi-spec.json +++ b/hosted-zone-api/abi-spec.json @@ -16,6 +16,23 @@ ], "fields": [] }, + "ai.vespa.cloud.Cloud": { + "superClass": "java.lang.Record", + "interfaces": [], + "attributes": [ + "public", + "final", + "record" + ], + "methods": [ + "public void <init>(java.lang.String)", + "public final java.lang.String toString()", + "public final int hashCode()", + "public final boolean equals(java.lang.Object)", + "public java.lang.String name()" + ], + "fields": [] + }, "ai.vespa.cloud.Cluster": { "superClass": "java.lang.Object", "interfaces": [], @@ -73,8 +90,10 @@ ], "methods": [ "public void <init>(ai.vespa.cloud.ApplicationId, ai.vespa.cloud.Zone, ai.vespa.cloud.Cluster, ai.vespa.cloud.Node)", + "public void <init>(ai.vespa.cloud.ApplicationId, ai.vespa.cloud.Zone, ai.vespa.cloud.Cloud, ai.vespa.cloud.Cluster, ai.vespa.cloud.Node)", "public ai.vespa.cloud.ApplicationId application()", "public ai.vespa.cloud.Zone zone()", + "public ai.vespa.cloud.Cloud cloud()", "public ai.vespa.cloud.Cluster cluster()", "public ai.vespa.cloud.Node node()" ], diff --git a/hosted-zone-api/src/main/java/ai/vespa/cloud/Cloud.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/Cloud.java new file mode 100644 index 00000000000..1281ffe7038 --- /dev/null +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/Cloud.java @@ -0,0 +1,11 @@ +package ai.vespa.cloud; + +/** + * The cloud provider in which a cloud deployment may be running. + * + * This is "aws" when this runs in Amazon Web Services, and "gcp" when this runs in Google Cloud Platform. + * + * @author mpolden + */ +public record Cloud(String name) { +} diff --git a/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java index 752e78b580f..325f62cd7f8 100644 --- a/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java @@ -13,12 +13,20 @@ public class SystemInfo { private final ApplicationId application; private final Zone zone; + private final Cloud cloud; private final Cluster cluster; private final Node node; + // TODO: Remove on Vespa 9 + @Deprecated(forRemoval = true) public SystemInfo(ApplicationId application, Zone zone, Cluster cluster, Node node) { + this(application, zone, new Cloud(""), cluster, node); + } + + public SystemInfo(ApplicationId application, Zone zone, Cloud cloud, Cluster cluster, Node node) { this.application = Objects.requireNonNull(application, "Application cannot be null"); this.zone = Objects.requireNonNull(zone, "Zone cannot be null"); + this.cloud = Objects.requireNonNull(cloud, "Cloud cannot be null"); this.cluster = Objects.requireNonNull(cluster, "Cluster cannot be null"); this.node = Objects.requireNonNull(node, "Node cannot be null"); } @@ -29,6 +37,11 @@ public class SystemInfo { /** Returns the zone this is running in */ public Zone zone() { return zone; } + /** Returns the cloud provider this is running in */ + public Cloud cloud() { + return cloud; + } + /** Returns the cluster this is part of */ public Cluster cluster() { return cluster; } diff --git a/hosted-zone-api/src/test/java/ai/vespa/cloud/SystemInfoTest.java b/hosted-zone-api/src/test/java/ai/vespa/cloud/SystemInfoTest.java index c14955d6614..80b22a185bb 100644 --- a/hosted-zone-api/src/test/java/ai/vespa/cloud/SystemInfoTest.java +++ b/hosted-zone-api/src/test/java/ai/vespa/cloud/SystemInfoTest.java @@ -17,12 +17,14 @@ public class SystemInfoTest { public void testSystemInfo() { ApplicationId application = new ApplicationId("tenant1", "application1", "instance1"); Zone zone = new Zone(Environment.dev, "us-west-1"); + Cloud cloud = new Cloud("aws"); Cluster cluster = new Cluster(1, List.of()); Node node = new Node(0); - SystemInfo info = new SystemInfo(application, zone, cluster, node); + SystemInfo info = new SystemInfo(application, zone, cloud, cluster, node); assertEquals(application, info.application()); assertEquals(zone, info.zone()); + assertEquals(cloud, info.cloud()); assertEquals(cluster, info.cluster()); assertEquals(node, info.node()); } diff --git a/jdisc-security-filters/pom.xml b/jdisc-security-filters/pom.xml index dcfb61de3cc..475a8b7e3e9 100644 --- a/jdisc-security-filters/pom.xml +++ b/jdisc-security-filters/pom.xml @@ -29,12 +29,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>security-utils</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> <!-- test --> <dependency> diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/csp/CspResponseFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/csp/CspResponseFilter.java new file mode 100644 index 00000000000..1d5388a2fb1 --- /dev/null +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/csp/CspResponseFilter.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.filter.security.csp; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.jdisc.AbstractResource; +import com.yahoo.jdisc.http.filter.DiscFilterResponse; +import com.yahoo.jdisc.http.filter.RequestView; +import com.yahoo.jdisc.http.filter.SecurityResponseFilter; +import com.yahoo.yolean.chain.Provides; + +/** + * The HTTP Content-Security-Policy (CSP) sandbox directive enables a sandbox for the requested resource similar to + * the <iframe> sandbox attribute. It applies restrictions to a page's actions including preventing popups, + * preventing the execution of plugins and scripts, and enforcing a same-origin policy. + * + * @author freva + */ +@Provides("CspResponseFilter") +public class CspResponseFilter extends AbstractResource implements SecurityResponseFilter { + + @Inject + public CspResponseFilter() { } + + @Override + public void filter(DiscFilterResponse response, RequestView request) { + response.setHeader("Content-Security-Policy", "sandbox"); + } + +} diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/csp/package-info.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/csp/package-info.java new file mode 100644 index 00000000000..c8784b32fcb --- /dev/null +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/csp/package-info.java @@ -0,0 +1,8 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author freva + */ +@ExportPackage +package com.yahoo.jdisc.http.filter.security.csp; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/SecurityHeadersResponseFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/SecurityHeadersResponseFilter.java index 21edd1c8e10..24cd9245b61 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/SecurityHeadersResponseFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/SecurityHeadersResponseFilter.java @@ -18,5 +18,6 @@ public class SecurityHeadersResponseFilter implements SecurityResponseFilter { response.setHeader("Pragma", "no-cache"); response.setHeader("Strict-Transport-Security", "max-age=31536000; includeSubDomains"); response.setHeader("X-Content-Type-Options", "nosniff"); + response.setHeader("X-Frame-Options", "DENY"); } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java index 75cca79f556..6b57b1e90e7 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java @@ -18,6 +18,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * A bundle {@link CollisionHook} that contains a set of bundles that are allowed to collide with bundles @@ -50,8 +51,10 @@ public class BundleCollisionHook implements CollisionHook, EventHook, FindHook { /** * Adds a collection of bundles to the allowed duplicates. + * Also clears any previous allowed duplicates of the new allowed duplicates. */ synchronized void allowDuplicateBundles(Collection<Bundle> bundles) { + allowedDuplicates.values().removeAll(bundles.stream().map(BsnVersion::new).collect(Collectors.toSet())); for (var bundle : bundles) { allowedDuplicates.put(bundle, new BsnVersion(bundle)); } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/UnsafeContentInputStream.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/UnsafeContentInputStream.java index 3855f053b76..731e28b556c 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/UnsafeContentInputStream.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/UnsafeContentInputStream.java @@ -19,7 +19,7 @@ import java.util.Objects; public class UnsafeContentInputStream extends InputStream { private final ReadableContentChannel content; - private ByteBuffer buf = ByteBuffer.allocate(0); + private ByteBuffer currBuf = ByteBuffer.allocate(0); private byte [] marked; private int readSinceMarked; @@ -34,13 +34,10 @@ public class UnsafeContentInputStream extends InputStream { @Override public int read() { - while (buf != null && buf.remaining() == 0) { - buf = content.read(); - } - if (buf == null) { - return -1; - } - byte b = buf.get(); + fetchNonEmptyBuffer(); + if (currBuf == null) return -1; + + byte b = currBuf.get(); if (marked != null) { if (readSinceMarked < marked.length) { marked[readSinceMarked++] = b; @@ -51,34 +48,45 @@ public class UnsafeContentInputStream extends InputStream { return ((int)b) & 0xFF; } + private boolean fetchNonEmptyBuffer() { + while (currBuf != null && currBuf.remaining() == 0) { + currBuf = content.read(); + } + return (currBuf != null && currBuf.hasRemaining()); + } + @Override public int read(byte buf[], int off, int len) { Objects.requireNonNull(buf, "buf"); if (off < 0 || len < 0 || len > buf.length - off) { throw new IndexOutOfBoundsException(); } - if (len == 0) { - return 0; - } - int c = read(); - if (c == -1) { - return -1; + if (len == 0) return 0; + + if ( ! fetchNonEmptyBuffer() ) return -1; + int read = 0; + while ((available() > 0) && fetchNonEmptyBuffer() && ((len - read) > 0)) { + int toRead = Math.min(currBuf.remaining(), (len - read)); + currBuf.get(buf, off + read, toRead); + read += toRead; } - buf[off] = (byte)c; - int cnt = 1; - for (; cnt < len && available() > 0; ++cnt) { - if ((c = read()) == -1) { - break; + if (marked != null) { + if (readSinceMarked + len < marked.length) { + for (int i=0; i < len; i++) { + marked[readSinceMarked++] = buf[off+i]; + } + } else { + marked = null; } - buf[off + cnt] = (byte)c; + } - return cnt; + return read; } @Override public int available() { - if (buf != null && buf.remaining() > 0) { - return buf.remaining(); + if (currBuf != null && currBuf.remaining() > 0) { + return currBuf.remaining(); } return content.available(); } @@ -102,11 +110,11 @@ public class UnsafeContentInputStream extends InputStream { if (marked == null) { throw new IOException("mark has not been called, or too much has been read since marked."); } - ByteBuffer newBuf = ByteBuffer.allocate(readSinceMarked + buf.remaining()); + ByteBuffer newBuf = ByteBuffer.allocate(readSinceMarked + currBuf.remaining()); newBuf.put(marked, 0, readSinceMarked); - newBuf.put(buf); + newBuf.put(currBuf); newBuf.flip(); - buf = newBuf; + currBuf = newBuf; marked = null; } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java index c457a703ecc..811d8a25459 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java @@ -199,7 +199,6 @@ public class RPCNetwork implements Network, MethodHandler { } this.owner = owner; - sendAdapters.put(new Version(5), new RPCSendV1(this)); sendAdapters.put(new Version(6,149), new RPCSendV2(this)); } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV1.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV1.java deleted file mode 100755 index 53318aa1299..00000000000 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV1.java +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.messagebus.network.rpc; - -import com.yahoo.component.Version; -import com.yahoo.jrt.DataValue; -import com.yahoo.jrt.DoubleValue; -import com.yahoo.jrt.Int32Array; -import com.yahoo.jrt.Int32Value; -import com.yahoo.jrt.Int64Value; -import com.yahoo.jrt.Int8Value; -import com.yahoo.jrt.Method; -import com.yahoo.jrt.Request; -import com.yahoo.jrt.StringArray; -import com.yahoo.jrt.StringValue; -import com.yahoo.jrt.Values; -import com.yahoo.messagebus.EmptyReply; -import com.yahoo.messagebus.Error; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.Reply; -import com.yahoo.messagebus.Trace; -import com.yahoo.messagebus.TraceNode; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.text.Utf8Array; - -/** - * Implements the request adapter for method "mbus.send1". - * - * @author Simon Thoresen Hult - */ -public class RPCSendV1 extends RPCSend { - - private final String METHOD_NAME = "mbus.send1"; - private final String METHOD_PARAMS = "sssbilsxi"; - private final String METHOD_RETURN = "sdISSsxs"; - - protected RPCSendV1(RPCNetwork net) { super(net); } - - @Override - protected String getReturnSpec() { return METHOD_RETURN; } - - @Override - protected Method buildMethod() { - - Method method = new Method(METHOD_NAME, METHOD_PARAMS, METHOD_RETURN, this); - method.methodDesc("Send a message bus request and get a reply back."); - method.paramDesc(0, "version", "The version of the message.") - .paramDesc(1, "route", "Names of additional hops to visit.") - .paramDesc(2, "session", "The local session that should receive this message.") - .paramDesc(3, "retryEnabled", "Whether or not this message can be resent.") - .paramDesc(4, "retry", "The number of times the sending of this message has been retried.") - .paramDesc(5, "timeRemaining", "The number of milliseconds until timeout.") - .paramDesc(6, "protocol", "The name of the protocol that knows how to decode this message.") - .paramDesc(7, "payload", "The protocol specific message payload.") - .paramDesc(8, "level", "The trace level of the message."); - method.returnDesc(0, "version", "The lowest version the message was serialized as.") - .returnDesc(1, "retryDelay", "The retry request of the reply.") - .returnDesc(2, "errorCodes", "The reply error codes.") - .returnDesc(3, "errorMessages", "The reply error messages.") - .returnDesc(4, "errorServices", "The reply error service names.") - .returnDesc(5, "protocol", "The name of the protocol that knows how to decode this reply.") - .returnDesc(6, "payload", "The protocol specific reply payload.") - .returnDesc(7, "trace", "A string representation of the trace."); - return method; - } - @Override - protected Request encodeRequest(Version version, Route route, RPCServiceAddress address, Message msg, - long timeRemaining, byte[] payload, int traceLevel) { - Request req = new Request(METHOD_NAME); - Values v = req.parameters(); - v.add(new StringValue(version.toUtf8())); - v.add(new StringValue(route.toString())); - v.add(new StringValue(address.getSessionName())); - v.add(new Int8Value(msg.getRetryEnabled() ? (byte)1 : (byte)0)); - v.add(new Int32Value(msg.getRetry())); - v.add(new Int64Value(timeRemaining)); - v.add(new StringValue(msg.getProtocol())); - v.add(new DataValue(payload)); - v.add(new Int32Value(traceLevel)); - return req; - } - - @Override - protected Reply createReply(Values ret, String serviceName, Trace trace) { - Version version = new Version(ret.get(0).asUtf8Array()); - double retryDelay = ret.get(1).asDouble(); - int[] errorCodes = ret.get(2).asInt32Array(); - String[] errorMessages = ret.get(3).asStringArray(); - String[] errorServices = ret.get(4).asStringArray(); - Utf8Array protocolName = ret.get(5).asUtf8Array(); - byte[] payload = ret.get(6).asData(); - String replyTrace = ret.get(7).asString(); - - // Make sure that the owner understands the protocol. - Reply reply = null; - Error error = null; - if (payload.length > 0) { - Object retval = decode(protocolName, version, payload); - if (retval instanceof Reply) { - reply = (Reply) retval; - } else { - error = (Error) retval; - } - } - if (reply == null) { - reply = new EmptyReply(); - } - if (error != null) { - reply.addError(error); - } - reply.setRetryDelay(retryDelay); - for (int i = 0; i < errorCodes.length && i < errorMessages.length; i++) { - reply.addError(new Error(errorCodes[i], errorMessages[i], - errorServices[i].length() > 0 ? errorServices[i] : serviceName)); - } - if (trace.getLevel() > 0) { - trace.getRoot().addChild(TraceNode.decode(replyTrace)); - } - return reply; - } - - protected Params toParams(Values args) { - Params p = new Params(); - p.version = new Version(args.get(0).asUtf8Array()); - p.route = args.get(1).asString(); - p.session = args.get(2).asString(); - p.retryEnabled = (args.get(3).asInt8() != 0); - p.retry = args.get(4).asInt32(); - p.timeRemaining = args.get(5).asInt64(); - p.protocolName = args.get(6).asUtf8Array(); - p.payload = args.get(7).asData(); - p.traceLevel = args.get(8).asInt32(); - return p; - } - - @Override - protected void createResponse(Values ret, Reply reply, Version version, byte [] payload) { - int[] eCodes = new int[reply.getNumErrors()]; - String[] eMessages = new String[reply.getNumErrors()]; - String[] eServices = new String[reply.getNumErrors()]; - for (int i = 0; i < reply.getNumErrors(); ++i) { - Error error = reply.getError(i); - eCodes[i] = error.getCode(); - eMessages[i] = error.getMessage(); - eServices[i] = error.getService() != null ? error.getService() : ""; - } - ret.add(new StringValue(version.toUtf8())); - ret.add(new DoubleValue(reply.getRetryDelay())); - ret.add(new Int32Array(eCodes)); - ret.add(new StringArray(eMessages)); - ret.add(new StringArray(eServices)); - ret.add(new StringValue(reply.getProtocol())); - ret.add(new DataValue(payload)); - ret.add(new StringValue(reply.getTrace().getRoot() != null ? reply.getTrace().getRoot().encode() : "")); - } - -} diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SendAdapterTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SendAdapterTestCase.java index efb1045ba84..437be44ea25 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SendAdapterTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SendAdapterTestCase.java @@ -4,7 +4,16 @@ package com.yahoo.messagebus.network.rpc; import com.yahoo.component.Version; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.DestinationSession; +import com.yahoo.messagebus.DestinationSessionParams; +import com.yahoo.messagebus.IntermediateSession; +import com.yahoo.messagebus.IntermediateSessionParams; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.Routable; +import com.yahoo.messagebus.SourceSession; +import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.test.TestServer; import com.yahoo.messagebus.routing.Route; @@ -20,7 +29,11 @@ import java.net.UnknownHostException; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + /** * @author Simon Thoresen Hult @@ -81,17 +94,15 @@ public class SendAdapterTestCase { @Test public void requireCorrectVersionSelection() { assertNull(srcServer.net.getSendAdapter(new Version(4,999))); - assertTrue(srcServer.net.getSendAdapter(new Version(5,0)) instanceof RPCSendV1); - assertTrue(srcServer.net.getSendAdapter(new Version(6,148)) instanceof RPCSendV1); + assertNull(srcServer.net.getSendAdapter(new Version(5,0))); + assertNull(srcServer.net.getSendAdapter(new Version(6,148))); assertTrue(srcServer.net.getSendAdapter(new Version(6,149)) instanceof RPCSendV2); assertTrue(srcServer.net.getSendAdapter(new Version(9,9999)) instanceof RPCSendV2); } @Test - public void requireThatMessagesCanBeSentAcrossAllSupportedVersions() throws Exception { + public void requireThatMessagesCanBeSentAcrossAllSupportedVersions() { List<Version> versions = Arrays.asList( - new Version(5, 0), - new Version(6, 148), new Version(6, 149), new Version(9, 999) ); diff --git a/messagebus/src/tests/messenger/messenger.cpp b/messagebus/src/tests/messenger/messenger.cpp index 1ee34b386ae..acd4406a7dc 100644 --- a/messagebus/src/tests/messenger/messenger.cpp +++ b/messagebus/src/tests/messenger/messenger.cpp @@ -39,7 +39,7 @@ public: TEST("messenger_test") { - Messenger msn(true, true); + Messenger msn; msn.start(); vespalib::Barrier barrier(2); diff --git a/messagebus/src/tests/protocolrepository/protocolrepository.cpp b/messagebus/src/tests/protocolrepository/protocolrepository.cpp index a9a268262ee..20649d56d13 100644 --- a/messagebus/src/tests/protocolrepository/protocolrepository.cpp +++ b/messagebus/src/tests/protocolrepository/protocolrepository.cpp @@ -13,42 +13,23 @@ private: public: - TestProtocol(const string &name) + TestProtocol(const string &name) noexcept : _name(name) - { - // empty - } + { } - const string & - getName() const override - { - return _name; - } + const string & getName() const override { return _name; } - IRoutingPolicy::UP - createPolicy(const string &name, const string ¶m) const override - { - (void)name; - (void)param; + IRoutingPolicy::UP createPolicy(const string &, const string &) const override { throw std::exception(); } - Blob - encode(const vespalib::Version &version, const Routable &routable) const override - { - (void)version; - (void)routable; + Blob encode(const vespalib::Version &, const Routable &) const override { throw std::exception(); } - Routable::UP - decode(const vespalib::Version &version, BlobRef data) const override - { - (void)version; - (void)data; + Routable::UP decode(const vespalib::Version &, BlobRef ) const override { throw std::exception(); } - bool requireSequencing() const override { return false; } }; int @@ -58,16 +39,16 @@ Test::Main() ProtocolRepository repo; IProtocol::SP prev; - prev = repo.putProtocol(IProtocol::SP(new TestProtocol("foo"))); - ASSERT_TRUE(prev.get() == NULL); + prev = repo.putProtocol(std::make_shared<TestProtocol>("foo")); + ASSERT_FALSE(prev); IRoutingPolicy::SP policy = repo.getRoutingPolicy("foo", "bar", "baz"); - prev = repo.putProtocol(IProtocol::SP(new TestProtocol("foo"))); - ASSERT_TRUE(prev.get() != NULL); + prev = repo.putProtocol(std::make_shared<TestProtocol>("foo")); + ASSERT_TRUE(prev); ASSERT_NOT_EQUAL(prev.get(), repo.getProtocol("foo")); policy = repo.getRoutingPolicy("foo", "bar", "baz"); - ASSERT_TRUE(policy.get() == NULL); + ASSERT_FALSE(policy); TEST_DONE(); } diff --git a/messagebus/src/tests/sendadapter/sendadapter.cpp b/messagebus/src/tests/sendadapter/sendadapter.cpp index 0ae957ea340..80bfc45184f 100644 --- a/messagebus/src/tests/sendadapter/sendadapter.cpp +++ b/messagebus/src/tests/sendadapter/sendadapter.cpp @@ -5,7 +5,6 @@ #include <vespa/messagebus/testlib/simplereply.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/messagebus/network/rpcsendv1.h> #include <vespa/messagebus/network/rpcsendv2.h> #include <vespa/vespalib/testkit/testapp.h> @@ -211,10 +210,8 @@ TEST("test that all known versions are present") { TestData data; ASSERT_TRUE(data.start()); EXPECT_FALSE(data._srcServer.net.getSendAdapter(vespalib::Version(4, 999)) != nullptr); - EXPECT_TRUE(data._srcServer.net.getSendAdapter(vespalib::Version(5, 0)) != nullptr); - EXPECT_TRUE(dynamic_cast<mbus::RPCSendV1 *>(data._srcServer.net.getSendAdapter(vespalib::Version(5, 0))) != nullptr); - EXPECT_TRUE(data._srcServer.net.getSendAdapter(vespalib::Version(6, 148)) != nullptr); - EXPECT_TRUE(dynamic_cast<mbus::RPCSendV1 *>(data._srcServer.net.getSendAdapter(vespalib::Version(6, 148))) != nullptr); + EXPECT_FALSE(data._srcServer.net.getSendAdapter(vespalib::Version(5, 0)) != nullptr); + EXPECT_FALSE(data._srcServer.net.getSendAdapter(vespalib::Version(6, 148)) != nullptr); EXPECT_TRUE(data._srcServer.net.getSendAdapter(vespalib::Version(6, 149)) != nullptr); EXPECT_TRUE(dynamic_cast<mbus::RPCSendV2 *>(data._srcServer.net.getSendAdapter(vespalib::Version(6, 149))) != nullptr); EXPECT_TRUE(data._srcServer.net.getSendAdapter(vespalib::Version(9, 999)) != nullptr); @@ -224,7 +221,7 @@ TEST("test that all known versions are present") { TEST("test that we can send between multiple versions") { TestData data; ASSERT_TRUE(data.start()); - TEST_DO(testSendAdapters(data, {vespalib::Version(5, 0), vespalib::Version(6, 148), vespalib::Version(6, 149), vespalib::Version(9, 999)})); + TEST_DO(testSendAdapters(data, {vespalib::Version(6, 149), vespalib::Version(9, 999)})); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/vespa/messagebus/iprotocol.h b/messagebus/src/vespa/messagebus/iprotocol.h index 20726ced71b..e46a5600471 100644 --- a/messagebus/src/vespa/messagebus/iprotocol.h +++ b/messagebus/src/vespa/messagebus/iprotocol.h @@ -79,9 +79,6 @@ public: * @return The decoded routable. */ virtual Routable::UP decode(const vespalib::Version &version, BlobRef data) const = 0; // throw() - - - virtual bool requireSequencing() const = 0; }; } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/messagebus.cpp b/messagebus/src/vespa/messagebus/messagebus.cpp index b4b29b84793..97be5955e9d 100644 --- a/messagebus/src/vespa/messagebus/messagebus.cpp +++ b/messagebus/src/vespa/messagebus/messagebus.cpp @@ -82,13 +82,13 @@ public: namespace mbus { -MessageBus::MessageBus(INetwork &net, ProtocolSet protocols, bool skip_request_thread, bool skip_reply_thread) : +MessageBus::MessageBus(INetwork &net, ProtocolSet protocols) : _network(net), _lock(), _routingTables(), _sessions(), _protocolRepository(std::make_unique<ProtocolRepository>()), - _msn(std::make_unique<Messenger>(skip_request_thread, skip_reply_thread)), + _msn(std::make_unique<Messenger>()), _resender(), _maxPendingCount(0), _maxPendingSize(0), @@ -111,7 +111,7 @@ MessageBus::MessageBus(INetwork &net, const MessageBusParams ¶ms) : _routingTables(), _sessions(), _protocolRepository(std::make_unique<ProtocolRepository>()), - _msn(std::make_unique<Messenger>(true, true)), + _msn(std::make_unique<Messenger>()), _resender(), _maxPendingCount(params.getMaxPendingCount()), _maxPendingSize(params.getMaxPendingSize()), diff --git a/messagebus/src/vespa/messagebus/messagebus.h b/messagebus/src/vespa/messagebus/messagebus.h index d2e82835f3e..d270a0f3491 100644 --- a/messagebus/src/vespa/messagebus/messagebus.h +++ b/messagebus/src/vespa/messagebus/messagebus.h @@ -82,7 +82,7 @@ public: * @param network The network to associate with. * @param protocols An array of protocols to register. */ - MessageBus(INetwork &net, ProtocolSet protocols, bool skip_request_thread, bool skip_reply_thread); + MessageBus(INetwork &net, ProtocolSet protocols); /** * Constructs an instance of message bus. This requires a network object that it will associate with. This diff --git a/messagebus/src/vespa/messagebus/messenger.cpp b/messagebus/src/vespa/messagebus/messenger.cpp index 13ca5317148..1423876e95b 100644 --- a/messagebus/src/vespa/messagebus/messenger.cpp +++ b/messagebus/src/vespa/messagebus/messenger.cpp @@ -155,14 +155,12 @@ public: namespace mbus { -Messenger::Messenger(bool skip_request_thread, bool skip_reply_thread) +Messenger::Messenger() : _lock(), _pool(128000), _children(), _queue(), - _closed(false), - _skip_request_thread(skip_request_thread), - _skip_reply_thread(skip_reply_thread) + _closed(false) {} Messenger::~Messenger() @@ -246,21 +244,13 @@ Messenger::start() void Messenger::deliverMessage(Message::UP msg, IMessageHandler &handler) { - if (_skip_request_thread) { - handler.handleMessage(std::move(msg)); - } else { - enqueue(std::make_unique<MessageTask>(std::move(msg), handler)); - } + handler.handleMessage(std::move(msg)); } void Messenger::deliverReply(Reply::UP reply, IReplyHandler &handler) { - if (_skip_reply_thread) { - handler.handleReply(std::move(reply)); - } else { - enqueue(std::make_unique<ReplyTask>(std::move(reply), handler)); - } + handler.handleReply(std::move(reply)); } void diff --git a/messagebus/src/vespa/messagebus/messenger.h b/messagebus/src/vespa/messagebus/messenger.h index be4dbdb10d8..0d36e6006cb 100644 --- a/messagebus/src/vespa/messagebus/messenger.h +++ b/messagebus/src/vespa/messagebus/messenger.h @@ -42,17 +42,15 @@ private: mutable std::mutex _lock; std::condition_variable _cond; FastOS_ThreadPool _pool; - std::vector<ITask*> _children; + std::vector<ITask*> _children; vespalib::ArrayQueue<ITask*> _queue; bool _closed; - const bool _skip_request_thread; - const bool _skip_reply_thread; protected: void Run(FastOS_ThreadInterface *thread, void *arg) override; public: - Messenger(bool skip_request_thread, bool skip_reply_thread); + Messenger(); /** * Frees any allocated resources. Also destroys all queued tasks. diff --git a/messagebus/src/vespa/messagebus/network/CMakeLists.txt b/messagebus/src/vespa/messagebus/network/CMakeLists.txt index fffef5518c9..8b058ef5fcc 100644 --- a/messagebus/src/vespa/messagebus/network/CMakeLists.txt +++ b/messagebus/src/vespa/messagebus/network/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_add_library(messagebus_network OBJECT rpcnetwork.cpp rpcnetworkparams.cpp rpcsend.cpp - rpcsendv1.cpp rpcsendv2.cpp rpcservice.cpp rpcserviceaddress.cpp diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index 0f7ebeb9a36..77d8ca24cfc 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "rpcnetwork.h" #include "rpcservicepool.h" -#include "rpcsendv1.h" #include "rpcsendv2.h" #include "rpctargetpool.h" #include "rpcnetworkparams.h" @@ -137,7 +136,6 @@ RPCNetwork::RPCNetwork(const RPCNetworkParams ¶ms) : _targetPoolTask(std::make_unique<TargetPoolTask>(_scheduler, *_targetPool)), _servicePool(std::make_unique<RPCServicePool>(*_mirror, 4_Ki)), _executor(std::make_unique<vespalib::ThreadStackExecutor>(params.getNumThreads(), 64_Ki)), - _sendV1(std::make_unique<RPCSendV1>()), _sendV2(std::make_unique<RPCSendV2>()), _sendAdapters(), _compressionConfig(params.getCompressionConfig()), @@ -195,9 +193,7 @@ RPCNetwork::attach(INetworkOwner &owner) LOG_ASSERT(_owner == nullptr); _owner = &owner; - _sendV1->attach(*this); _sendV2->attach(*this); - _sendAdapters[vespalib::Version(5)] = _sendV1.get(); _sendAdapters[vespalib::Version(6, 149)] = _sendV2.get(); FRT_ReflectionBuilder builder(_orb.get()); diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.h b/messagebus/src/vespa/messagebus/network/rpcnetwork.h index 38f35d1266a..e706431f90d 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.h @@ -66,7 +66,6 @@ private: std::unique_ptr<FNET_Task> _targetPoolTask; std::unique_ptr<RPCServicePool> _servicePool; std::unique_ptr<vespalib::SyncableThreadExecutor> _executor; - std::unique_ptr<RPCSendAdapter> _sendV1; std::unique_ptr<RPCSendAdapter> _sendV2; SendAdapterMap _sendAdapters; CompressionConfig _compressionConfig; diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h index 01834074e6f..16b8b9b1570 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h @@ -218,20 +218,6 @@ public: } bool getDispatchOnEncode() const { return _dispatchOnEncode; } - - RPCNetworkParams &setSkipRequestThread(bool skip_request_thread) { - _skip_request_thread = skip_request_thread; - return *this; - } - - bool getSkipRequestThread() const { return _skip_request_thread; } - - RPCNetworkParams &setSkipReplyThread(bool skip_reply_thread) { - _skip_reply_thread = skip_reply_thread; - return *this; - } - - bool getSkipReplyThread() const { return _skip_reply_thread; } }; } diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.cpp b/messagebus/src/vespa/messagebus/network/rpcsend.cpp index 485c8e3e911..7627aa876b3 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsend.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsend.cpp @@ -55,10 +55,10 @@ private: } -RPCSend::RPCSend() : - _net(nullptr), - _clientIdent("client"), - _serverIdent("server") +RPCSend::RPCSend() + : _net(nullptr), + _clientIdent("client"), + _serverIdent("server") { } RPCSend::~RPCSend() = default; @@ -220,19 +220,19 @@ RPCSend::decode(vespalib::stringref protocolName, const vespalib::Version & vers void RPCSend::handleReply(Reply::UP reply) { - const IProtocol * protocol = _net->getOwner().getProtocol(reply->getProtocol()); - if (!protocol || protocol->requireSequencing() || !_net->allowDispatchForEncode()) { - doHandleReply(protocol, std::move(reply)); + if (!_net->allowDispatchForEncode()) { + doHandleReply(std::move(reply)); } else { - auto rejected = _net->getExecutor().execute(makeLambdaTask([this, protocol, reply = std::move(reply)]() mutable { - doHandleReply(protocol, std::move(reply)); + auto rejected = _net->getExecutor().execute(makeLambdaTask([this, reply = std::move(reply)]() mutable { + doHandleReply(std::move(reply)); })); assert (!rejected); } } void -RPCSend::doHandleReply(const IProtocol * protocol, Reply::UP reply) { +RPCSend::doHandleReply(Reply::UP reply) { + const IProtocol * protocol = _net->getOwner().getProtocol(reply->getProtocol()); ReplyContext::UP ctx(static_cast<ReplyContext*>(reply->getContext().value.PTR)); FRT_RPCRequest &req = ctx->getRequest(); string version = ctx->getVersion().toString(); @@ -256,29 +256,29 @@ void RPCSend::invoke(FRT_RPCRequest *req) { req->Detach(); - FRT_Values &args = *req->GetParams(); - std::unique_ptr<Params> params = toParams(args); - IProtocol * protocol = _net->getOwner().getProtocol(params->getProtocol()); - if (protocol == nullptr) { - replyError(req, params->getVersion(), params->getTraceLevel(), - Error(ErrorCode::UNKNOWN_PROTOCOL, make_string("Protocol '%s' is not known by %s.", - vespalib::string(params->getProtocol()).c_str(), _serverIdent.c_str()))); - return; - } - if (protocol->requireSequencing() || !_net->allowDispatchForDecode()) { - doRequest(req, protocol, std::move(params)); + if (!_net->allowDispatchForDecode()) { + doRequest(req); } else { - auto rejected = _net->getExecutor().execute(makeLambdaTask([this, req, protocol, params = std::move(params)]() mutable { - doRequest(req, protocol, std::move(params)); + auto rejected = _net->getExecutor().execute(makeLambdaTask([this, req]() { + doRequest(req); })); assert (!rejected); } } void -RPCSend::doRequest(FRT_RPCRequest *req, const IProtocol * protocol, std::unique_ptr<Params> params) +RPCSend::doRequest(FRT_RPCRequest *req) { + FRT_Values &args = *req->GetParams(); + std::unique_ptr<Params> params = toParams(args); + IProtocol * protocol = _net->getOwner().getProtocol(params->getProtocol()); + if (protocol == nullptr) { + replyError(req, params->getVersion(), params->getTraceLevel(), + Error(ErrorCode::UNKNOWN_PROTOCOL, make_string("Protocol '%s' is not known by %s.", + vespalib::string(params->getProtocol()).c_str(), _serverIdent.c_str()))); + return; + } Routable::UP routable = protocol->decode(params->getVersion(), params->getPayload()); req->DiscardBlobs(); if ( ! routable ) { diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.h b/messagebus/src/vespa/messagebus/network/rpcsend.h index e548c4adca2..c2bcb7dff2b 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsend.h +++ b/messagebus/src/vespa/messagebus/network/rpcsend.h @@ -83,9 +83,9 @@ public: void invoke(FRT_RPCRequest *req); private: - void doRequest(FRT_RPCRequest *req, const IProtocol * protocol, std::unique_ptr<Params> params); + void doRequest(FRT_RPCRequest *req); void doRequestDone(FRT_RPCRequest *req); - void doHandleReply(const IProtocol * protocol, std::unique_ptr<Reply> reply); + void doHandleReply(std::unique_ptr<Reply> reply); void attach(RPCNetwork &net) final override; void handleDiscard(Context ctx) final override; void sendByHandover(RoutingNode &recipient, const vespalib::Version &version, diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv1.cpp b/messagebus/src/vespa/messagebus/network/rpcsendv1.cpp deleted file mode 100644 index 7342c264bfa..00000000000 --- a/messagebus/src/vespa/messagebus/network/rpcsendv1.cpp +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "rpcsendv1.h" -#include "rpcnetwork.h" -#include "rpcserviceaddress.h" -#include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/error.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/fnet/frt/reflection.h> - -using vespalib::make_string; - -namespace mbus { - -namespace { - -const char *METHOD_NAME = "mbus.send1"; -const char *METHOD_PARAMS = "sssbilsxi"; -const char *METHOD_RETURN = "sdISSsxs"; - -} - -bool RPCSendV1::isCompatible(vespalib::stringref method, vespalib::stringref request, vespalib::stringref respons) -{ - return (method == METHOD_NAME) && - (request == METHOD_PARAMS) && - (respons == METHOD_RETURN); -} - -const char * -RPCSendV1::getReturnSpec() const { - return METHOD_RETURN; -} - -void -RPCSendV1::build(FRT_ReflectionBuilder & builder) -{ - builder.DefineMethod(METHOD_NAME, METHOD_PARAMS, METHOD_RETURN, FRT_METHOD(RPCSendV1::invoke), this); - builder.MethodDesc("Send a message bus request and get a reply back."); - builder.ParamDesc("version", "The version of the message."); - builder.ParamDesc("route", "Names of additional hops to visit."); - builder.ParamDesc("session", "The local session that should receive this message."); - builder.ParamDesc("retryEnabled", "Whether or not this message can be resent."); - builder.ParamDesc("retry", "The number of times the sending of this message has been retried."); - builder.ParamDesc("timeRemaining", "The number of milliseconds until timeout."); - builder.ParamDesc("protocol", "The name of the protocol that knows how to decode this message."); - builder.ParamDesc("payload", "The protocol specific message payload."); - builder.ParamDesc("level", "The trace level of the message."); - builder.ReturnDesc("version", "The lowest version the message was serialized as."); - builder.ReturnDesc("retry", "The retry request of the reply."); - builder.ReturnDesc("errorCodes", "The reply error codes."); - builder.ReturnDesc("errorMessages", "The reply error messages."); - builder.ReturnDesc("errorServices", "The reply error service names."); - builder.ReturnDesc("protocol", "The name of the protocol that knows how to decode this reply."); - builder.ReturnDesc("payload", "The protocol specific reply payload."); - builder.ReturnDesc("trace", "A string representation of the trace."); -} - -void -RPCSendV1::encodeRequest(FRT_RPCRequest &req, const vespalib::Version &version, const Route & route, - const RPCServiceAddress & address, const Message & msg, uint32_t traceLevel, - const PayLoadFiller &filler, duration timeRemaining) const -{ - - FRT_Values &args = *req.GetParams(); - req.SetMethodName(METHOD_NAME); - args.AddString(version.toString().c_str()); - args.AddString(route.toString().c_str()); - args.AddString(address.getSessionName().c_str()); - args.AddInt8(msg.getRetryEnabled() ? 1 : 0); - args.AddInt32(msg.getRetry()); - args.AddInt64(vespalib::count_ms(timeRemaining)); - args.AddString(msg.getProtocol().c_str()); - filler.fill(args); - args.AddInt32(traceLevel); -} - -namespace { - -class ParamsV1 : public RPCSend::Params -{ -public: - ParamsV1(const FRT_Values &args) : _args(args) { } - - uint32_t getTraceLevel() const override { return _args[8]._intval32; } - bool useRetry() const override { return _args[3]._intval8 != 0; } - uint32_t getRetries() const override { return _args[4]._intval32; } - duration getRemainingTime() const override { return std::chrono::milliseconds(_args[5]._intval64); } - - vespalib::Version getVersion() const override { - return vespalib::Version(vespalib::stringref(_args[0]._string._str, _args[0]._string._len)); - } - vespalib::stringref getRoute() const override { - return vespalib::stringref(_args[1]._string._str, _args[1]._string._len); - } - vespalib::stringref getSession() const override { - return vespalib::stringref(_args[2]._string._str, _args[2]._string._len); - } - vespalib::stringref getProtocol() const override { - return vespalib::stringref(_args[6]._string._str, _args[6]._string._len); - } - BlobRef getPayload() const override { - return BlobRef(_args[7]._data._buf, _args[7]._data._len); - } -private: - const FRT_Values & _args; -}; - -} - -std::unique_ptr<RPCSend::Params> -RPCSendV1::toParams(const FRT_Values &args) const -{ - return std::make_unique<ParamsV1>(args); -} - - -std::unique_ptr<Reply> -RPCSendV1::createReply(const FRT_Values & ret, const string & serviceName, Error & error, vespalib::Trace & trace) const -{ - vespalib::Version version = vespalib::Version(ret[0]._string._str); - double retryDelay = ret[1]._double; - uint32_t *errorCodes = ret[2]._int32_array._pt; - uint32_t errorCodesLen = ret[2]._int32_array._len; - FRT_StringValue *errorMessages = ret[3]._string_array._pt; - uint32_t errorMessagesLen = ret[3]._string_array._len; - FRT_StringValue *errorServices = ret[4]._string_array._pt; - uint32_t errorServicesLen = ret[4]._string_array._len; - const char *protocolName = ret[5]._string._str; - BlobRef payload(ret[6]._data._buf, ret[6]._data._len); - const char *traceStr = ret[7]._string._str; - - Reply::UP reply; - if (payload.size() > 0) { - reply = decode(protocolName, version, payload, error); - } - if ( ! reply ) { - reply = std::make_unique<EmptyReply>(); - } - reply->setRetryDelay(retryDelay); - for (uint32_t i = 0; i < errorCodesLen && i < errorMessagesLen && i < errorServicesLen; ++i) { - reply->addError(Error(errorCodes[i], errorMessages[i]._str, - errorServices[i]._len > 0 ? errorServices[i]._str : serviceName.c_str())); - } - trace.addChild(TraceNode::decode(traceStr)); - return reply; -} - -void -RPCSendV1::createResponse(FRT_Values & ret, const string & version, Reply & reply, Blob payload) const { - ret.AddString(version.c_str()); - ret.AddDouble(reply.getRetryDelay()); - - uint32_t errorCount = reply.getNumErrors(); - uint32_t *errorCodes = ret.AddInt32Array(errorCount); - FRT_StringValue *errorMessages = ret.AddStringArray(errorCount); - FRT_StringValue *errorServices = ret.AddStringArray(errorCount); - for (uint32_t i = 0; i < errorCount; ++i) { - errorCodes[i] = reply.getError(i).getCode(); - ret.SetString(errorMessages + i, reply.getError(i).getMessage().c_str()); - ret.SetString(errorServices + i, reply.getError(i).getService().c_str()); - } - - ret.AddString(reply.getProtocol().c_str()); - ret.AddData(std::move(payload.payload()), payload.size()); - if (reply.getTrace().getLevel() > 0) { - ret.AddString(reply.getTrace().encode().c_str()); - } else { - ret.AddString(""); - } -} - -} // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv1.h b/messagebus/src/vespa/messagebus/network/rpcsendv1.h deleted file mode 100644 index 7010df9dde5..00000000000 --- a/messagebus/src/vespa/messagebus/network/rpcsendv1.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "rpcsend.h" - -namespace mbus { - -class RPCSendV1 : public RPCSend { -public: - static bool isCompatible(vespalib::stringref method, vespalib::stringref request, vespalib::stringref respons); -private: - void build(FRT_ReflectionBuilder & builder) override; - const char * getReturnSpec() const override; - std::unique_ptr<Params> toParams(const FRT_Values ¶m) const override; - void encodeRequest(FRT_RPCRequest &req, const vespalib::Version &version, const Route & route, - const RPCServiceAddress & address, const Message & msg, uint32_t traceLevel, - const PayLoadFiller &filler, duration timeRemaining) const override; - - std::unique_ptr<Reply> createReply(const FRT_Values & response, const string & serviceName, - Error & error, vespalib::Trace & trace) const override; - void createResponse(FRT_Values & ret, const string & version, Reply & reply, Blob payload) const override; -}; - -} // namespace mbus diff --git a/messagebus/src/vespa/messagebus/rpcmessagebus.cpp b/messagebus/src/vespa/messagebus/rpcmessagebus.cpp index a79171f5af2..0e4b6673d20 100644 --- a/messagebus/src/vespa/messagebus/rpcmessagebus.cpp +++ b/messagebus/src/vespa/messagebus/rpcmessagebus.cpp @@ -25,7 +25,7 @@ RPCMessageBus::RPCMessageBus(const ProtocolSet &protocols, const RPCNetworkParams &rpcParams, const config::ConfigUri &routingCfgUri) : _net(rpcParams), - _bus(_net, protocols, rpcParams.getSkipRequestThread(), rpcParams.getSkipReplyThread()), + _bus(_net, protocols), _agent(_bus), _subscriber(routingCfgUri.getContext()) { diff --git a/messagebus/src/vespa/messagebus/testlib/simpleprotocol.h b/messagebus/src/vespa/messagebus/testlib/simpleprotocol.h index 57a6e901b90..0486f058217 100644 --- a/messagebus/src/vespa/messagebus/testlib/simpleprotocol.h +++ b/messagebus/src/vespa/messagebus/testlib/simpleprotocol.h @@ -72,7 +72,6 @@ public: IRoutingPolicy::UP createPolicy(const string &name, const string ¶m) const override; Blob encode(const vespalib::Version &version, const Routable &routable) const override; Routable::UP decode(const vespalib::Version &version, BlobRef data) const override; - virtual bool requireSequencing() const override { return false; } }; } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/testlib/testserver.cpp b/messagebus/src/vespa/messagebus/testlib/testserver.cpp index 80d9eaefe29..d289c372fda 100644 --- a/messagebus/src/vespa/messagebus/testlib/testserver.cpp +++ b/messagebus/src/vespa/messagebus/testlib/testserver.cpp @@ -25,7 +25,7 @@ TestServer::TestServer(const Identity &ident, const Slobrok &slobrok, IProtocol::SP protocol) : net(RPCNetworkParams(slobrok.config()).setIdentity(ident)), - mb(net, ProtocolSet().add(std::make_shared<SimpleProtocol>()).add(protocol), true, true) + mb(net, ProtocolSet().add(std::make_shared<SimpleProtocol>()).add(protocol)) { mb.setupRouting(spec); } diff --git a/metrics-proxy/CMakeLists.txt b/metrics-proxy/CMakeLists.txt index 5963cba9615..4cc59300058 100644 --- a/metrics-proxy/CMakeLists.txt +++ b/metrics-proxy/CMakeLists.txt @@ -1,6 +1,8 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_jar(metrics-proxy-jar-with-dependencies.jar) +install(DIRECTORY DESTINATION conf/telegraf) +install(DIRECTORY DESTINATION logs/telegraf) vespa_install_script(src/main/sh/start-telegraf.sh libexec/vespa) vespa_install_script(src/main/sh/stop-telegraf.sh libexec/vespa) diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index b41e9d5c8a8..790b3298a81 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -1,25 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; -import ai.vespa.util.http.hc5.VespaAsyncHttpClientBuilder; +import ai.vespa.util.http.hc5.VespaHttpClientBuilder; import com.yahoo.yolean.Exceptions; +import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.config.RequestConfig; -import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; -import org.apache.hc.core5.http.ContentType; -import org.apache.hc.core5.http.HttpResponse; -import org.apache.hc.core5.http.Message; -import org.apache.hc.core5.http.Method; -import org.apache.hc.core5.http.nio.support.BasicRequestProducer; -import org.apache.hc.core5.http.nio.support.BasicResponseConsumer; -import org.apache.hc.core5.http.nio.support.classic.AbstractClassicEntityConsumer; -import org.apache.hc.core5.reactor.IOReactorConfig; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; +import org.apache.hc.core5.http.io.SocketConfig; import org.apache.hc.core5.util.Timeout; import java.io.IOException; -import java.io.InputStream; import java.net.URI; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; import java.util.logging.Level; import java.util.logging.Logger; @@ -36,10 +29,10 @@ public abstract class HttpMetricFetcher { // The call to apache will do 3 retries. As long as we check the services in series, we can't have this too high. public static volatile int CONNECTION_TIMEOUT = 5000; private final static int SOCKET_TIMEOUT = 60000; - private final static int BUFFER_SIZE = 0x40000; // 256k + final static int BUFFER_SIZE = 0x40000; // 256k private final URI url; protected final VespaService service; - private static final CloseableHttpAsyncClient httpClient = createHttpClient(); + private static final CloseableHttpClient httpClient = createHttpClient(); /** * @param service the service to fetch metrics from @@ -53,17 +46,9 @@ public abstract class HttpMetricFetcher { log.log(Level.FINE, () -> "Fetching metrics from " + u + " with timeout " + CONNECTION_TIMEOUT); } - InputStream getJson() throws IOException,InterruptedException, ExecutionException { + CloseableHttpResponse getResponse() throws IOException { log.log(Level.FINE, () -> "Connecting to url " + url + " for service '" + service + "'"); - Future<Message<HttpResponse, InputStream>> response = httpClient.execute( - new BasicRequestProducer(Method.GET, url), - new BasicResponseConsumer<>(new AbstractClassicEntityConsumer<>(BUFFER_SIZE, Runnable::run) { - @Override - protected InputStream consumeData(ContentType contentType, InputStream inputStream) { - return inputStream; - } - }), null); - return response.get().getBody(); + return httpClient.execute(new HttpGet(url)); } public String toString() { @@ -95,20 +80,21 @@ public abstract class HttpMetricFetcher { } } - private static CloseableHttpAsyncClient createHttpClient() { - CloseableHttpAsyncClient client = VespaAsyncHttpClientBuilder.create() + private static CloseableHttpClient createHttpClient() { + return VespaHttpClientBuilder.create(registry -> { + var mgr = new PoolingHttpClientConnectionManager(registry); + mgr.setDefaultSocketConfig(SocketConfig.custom() + .setSoTimeout(Timeout.ofMilliseconds(SOCKET_TIMEOUT)) + .build()); + return mgr; + }) .setUserAgent("metrics-proxy-http-client") - .setIOReactorConfig(IOReactorConfig.custom() - .setSoTimeout(Timeout.ofMilliseconds(SOCKET_TIMEOUT)) - .build()) .setDefaultRequestConfig(RequestConfig.custom() - .setConnectionRequestTimeout(Timeout.ofMilliseconds(SOCKET_TIMEOUT)) - .setConnectTimeout(Timeout.ofMilliseconds(CONNECTION_TIMEOUT)) - .setResponseTimeout(Timeout.ofMilliseconds(SOCKET_TIMEOUT)) - .build()) + .setConnectionRequestTimeout(Timeout.ofMilliseconds(SOCKET_TIMEOUT)) + .setConnectTimeout(Timeout.ofMilliseconds(CONNECTION_TIMEOUT)) + .setResponseTimeout(Timeout.ofMilliseconds(SOCKET_TIMEOUT)) + .build()) .build(); - client.start(); - return client; } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java index cb8e07b0282..f13fa8eef65 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java @@ -4,12 +4,14 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.HealthMetric; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import java.io.BufferedInputStream; +import java.io.IOException; import java.io.InputStream; -import java.util.concurrent.ExecutionException; import java.util.logging.Level; - -import java.io.IOException; import java.util.logging.Logger; /** @@ -31,9 +33,17 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { * Connect to remote service over http and fetch metrics */ public HealthMetric getHealth(int fetchCount) { - try (InputStream stream = getJson()) { - return createHealthMetrics(stream, fetchCount); - } catch (IOException | InterruptedException | ExecutionException e) { + try (CloseableHttpResponse response = getResponse()) { + HttpEntity entity = response.getEntity(); + try { + return parse(new BufferedInputStream(entity.getContent(), HttpMetricFetcher.BUFFER_SIZE)); + } catch (Exception e) { + handleException(e, entity.getContentType(), fetchCount); + return HealthMetric.getDown("Failed fetching status page for service"); + } finally { + EntityUtils.consumeQuietly(entity); + } + } catch (IOException e) { if (service.isAlive()) { logMessageNoResponse(errMsgNoResponse(e), fetchCount); } @@ -41,19 +51,6 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { } } - /** - * Connect to remote service over http and fetch metrics - */ - private HealthMetric createHealthMetrics(InputStream data, int fetchCount) throws IOException { - try { - return parse(data); - } catch (Exception e) { - handleException(e, data, fetchCount); - while (data.read() != -1) {} - return HealthMetric.getDown("Failed fetching status page for service"); - } - } - private HealthMetric parse(InputStream data) { try { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java index 5ca9e6fd950..cd05945f4f1 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java @@ -1,9 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.io.entity.EntityUtils; + +import java.io.BufferedInputStream; import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.ExecutionException; /** * Fetch metrics for a given vespa service @@ -22,21 +25,19 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { * Connect to remote service over http and fetch metrics */ public void getMetrics(MetricsParser.Consumer consumer, int fetchCount) { - try (InputStream stream = getJson()) { - createMetrics(stream, consumer, fetchCount); - } catch (IOException | InterruptedException | ExecutionException e) { - } + try (CloseableHttpResponse response = getResponse()) { + HttpEntity entity = response.getEntity(); + try { + MetricsParser.parse(new BufferedInputStream(entity.getContent(), HttpMetricFetcher.BUFFER_SIZE), consumer); + } catch (Exception e) { + handleException(e, entity.getContentType(), fetchCount); + } finally { + EntityUtils.consumeQuietly(entity); + } + } catch (IOException ignored) {} } void createMetrics(String data, MetricsParser.Consumer consumer, int fetchCount) throws IOException { MetricsParser.parse(data, consumer); } - private void createMetrics(InputStream data, MetricsParser.Consumer consumer, int fetchCount) throws IOException { - try { - MetricsParser.parse(data, consumer); - } catch (Exception e) { - handleException(e, data, fetchCount); - while (data.read() != -1) {} - } - } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java index cf6de804053..71a4466ea95 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java @@ -10,7 +10,6 @@ import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -43,7 +42,6 @@ public class RpcHealthMetricsTest { public Timeout globalTimeout = Timeout.seconds(300); @Test - @Ignore("Temporarily ignore test until timeout issue is resolved") public void expected_response_is_returned() { try (IntegrationTester tester = new IntegrationTester()) { MockHttpServer mockHttpServer = tester.httpServer(); @@ -69,7 +67,6 @@ public class RpcHealthMetricsTest { } @Test - @Ignore("Temporarily ignore test until timeout issue is resolved") public void non_existent_service_name_returns_an_error_message() { try (IntegrationTester tester = new IntegrationTester()) { String jsonRPCMessage = getHealthMetrics(tester, "non-existing service"); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java index 83149ad5ef7..a52e1daf878 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java @@ -14,7 +14,6 @@ import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -22,10 +21,10 @@ import org.junit.rules.Timeout; import java.io.IOException; import java.util.List; -import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.TestUtil.getFileContents; import static ai.vespa.metricsproxy.core.VespaMetrics.vespaMetricsConsumerId; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; +import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.rpc.IntegrationTester.CUSTOM_CONSUMER_ID; import static ai.vespa.metricsproxy.rpc.IntegrationTester.MONITORING_SYSTEM; import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_1_CONFIG_ID; @@ -68,7 +67,6 @@ public class RpcMetricsTest { public Timeout globalTimeout = Timeout.seconds(300); @Test - @Ignore("Temporarily ignore test until timeout issue is resolved") public void extra_metrics_are_added_to_output() throws Exception { try (IntegrationTester tester = new IntegrationTester()) { try (RpcClient rpcClient = new RpcClient(tester.rpcPort())) { @@ -84,7 +82,6 @@ public class RpcMetricsTest { } @Test - @Ignore("Temporarily ignore test until timeout issue is resolved") public void extra_metrics_are_purged() throws Exception { try (IntegrationTester tester = new IntegrationTester()) { try (RpcClient rpcClient = new RpcClient(tester.rpcPort())) { @@ -102,7 +99,6 @@ public class RpcMetricsTest { } @Test - @Ignore("Temporarily ignore test until timeout issue is resolved") public void testGetMetrics() throws Exception { try (IntegrationTester tester = new IntegrationTester()) { tester.httpServer().setResponse(METRICS_RESPONSE); @@ -193,7 +189,6 @@ public class RpcMetricsTest { } @Test - @Ignore("Temporarily ignore test until timeout issue is resolved") public void testGetAllMetricNames() { try (IntegrationTester tester = new IntegrationTester()) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index cad6b7b76ac..6295765a95f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.container.metrics.Dimensions; import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; +import com.yahoo.vespa.hosted.node.admin.maintenance.sync.ZstdCompressingInputStream; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; @@ -13,10 +14,11 @@ import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Clock; -import java.time.Duration; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -26,7 +28,6 @@ import java.util.UUID; import java.util.function.Supplier; import java.util.logging.Logger; import java.util.regex.Pattern; -import java.util.stream.Collectors; import java.util.stream.IntStream; import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameEndsWith; @@ -42,15 +43,14 @@ import static com.yahoo.yolean.Exceptions.uncheck; public class CoredumpHandler { private static final Pattern HS_ERR_PATTERN = Pattern.compile("hs_err_pid[0-9]+\\.log"); - private static final String LZ4_PATH = "/usr/bin/lz4"; private static final String PROCESSING_DIRECTORY_NAME = "processing"; private static final String METADATA_FILE_NAME = "metadata.json"; + private static final String COMPRESSED_EXTENSION = ".zstd"; public static final String COREDUMP_FILENAME_PREFIX = "dump_"; private final Logger logger = Logger.getLogger(CoredumpHandler.class.getName()); private final ObjectMapper objectMapper = new ObjectMapper(); - private final Terminal terminal; private final CoreCollector coreCollector; private final CoredumpReporter coredumpReporter; private final String crashPatchInContainer; @@ -65,14 +65,13 @@ public class CoredumpHandler { */ public CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter, String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics) { - this(terminal, coreCollector, coredumpReporter, crashPathInContainer, doneCoredumpsPath, + this(coreCollector, coredumpReporter, crashPathInContainer, doneCoredumpsPath, metrics, Clock.systemUTC(), () -> UUID.randomUUID().toString()); } - CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter, + CoredumpHandler(CoreCollector coreCollector, CoredumpReporter coredumpReporter, String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics, Clock clock, Supplier<String> coredumpIdSupplier) { - this.terminal = terminal; this.coreCollector = coreCollector; this.coredumpReporter = coredumpReporter; this.crashPatchInContainer = crashPathInContainer; @@ -94,7 +93,7 @@ public class CoredumpHandler { .match(fileAttributes -> !isReadyForProcessing(fileAttributes)) .maxDepth(1).stream() .map(FileFinder.FileAttributes::filename) - .collect(Collectors.toUnmodifiableList()); + .toList(); if (!pendingCores.isEmpty()) throw new ConvergenceException(String.format("Cannot process %s coredumps: Still being written", pendingCores.size() < 5 ? pendingCores : pendingCores.size())); @@ -130,7 +129,7 @@ public class CoredumpHandler { .stream() .sorted(Comparator.comparing(FileFinder.FileAttributes::lastModifiedTime)) .map(FileFinder.FileAttributes::path) - .collect(Collectors.toList()); + .toList(); int coredumpIndex = IntStream.range(0, toProcess.size()) .filter(i -> !HS_ERR_PATTERN.matcher(toProcess.get(i).getFileName().toString()).matches()) @@ -191,11 +190,14 @@ public class CoredumpHandler { */ private void finishProcessing(NodeAgentContext context, ContainerPath coredumpDirectory) throws IOException { ContainerPath coreFile = findCoredumpFileInProcessingDirectory(coredumpDirectory); - ContainerPath compressedCoreFile = coreFile.resolveSibling(coreFile.getFileName() + ".lz4"); - terminal.newCommandLine(context) - .add(LZ4_PATH, "-f", coreFile.pathOnHost().toString(), compressedCoreFile.pathOnHost().toString()) - .setTimeout(Duration.ofMinutes(30)) - .execute(); + ContainerPath compressedCoreFile = coreFile.resolveSibling(coreFile.getFileName() + COMPRESSED_EXTENSION); + + try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(Files.newInputStream(coreFile)); + OutputStream fos = Files.newOutputStream(compressedCoreFile)) { + zcis.transferTo(fos); + } catch (IOException e) { + throw new UncheckedIOException(e); + } Files.delete(coreFile); Path newCoredumpDirectory = doneCoredumpsPath.resolve(context.containerName().asString()); @@ -206,7 +208,7 @@ public class CoredumpHandler { ContainerPath findCoredumpFileInProcessingDirectory(ContainerPath coredumpProccessingDirectory) { return (ContainerPath) FileFinder.files(coredumpProccessingDirectory) - .match(nameStartsWith(COREDUMP_FILENAME_PREFIX).and(nameEndsWith(".lz4").negate())) + .match(nameStartsWith(COREDUMP_FILENAME_PREFIX).and(nameEndsWith(COMPRESSED_EXTENSION).negate())) .maxDepth(1) .stream() .map(FileFinder.FileAttributes::path) @@ -222,7 +224,7 @@ public class CoredumpHandler { int numberOfUnprocessedCoredumps = FileFinder.files(containerCrashPath) .match(nameStartsWith(".").negate()) .match(nameMatches(HS_ERR_PATTERN).negate()) - .match(nameEndsWith(".lz4").negate()) + .match(nameEndsWith(COMPRESSED_EXTENSION).negate()) .match(nameStartsWith("metadata").negate()) .list().size(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index 8f671910f93..5c681ddc129 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -8,10 +8,7 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2; -import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -51,14 +48,13 @@ public class CoredumpHandlerTest { private final ContainerPath containerCrashPath = context.paths().of("/var/crash"); private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps"); - private final TestTerminal terminal = new TestTerminal(); private final CoreCollector coreCollector = mock(CoreCollector.class); private final CoredumpReporter coredumpReporter = mock(CoredumpReporter.class); private final Metrics metrics = new Metrics(); private final ManualClock clock = new ManualClock(); @SuppressWarnings("unchecked") private final Supplier<String> coredumpIdSupplier = mock(Supplier.class); - private final CoredumpHandler coredumpHandler = new CoredumpHandler(terminal, coreCollector, coredumpReporter, + private final CoredumpHandler coredumpHandler = new CoredumpHandler(coreCollector, coredumpReporter, containerCrashPath.pathInContainer(), doneCoredumpsPath, metrics, clock, coredumpIdSupplier); @@ -187,7 +183,7 @@ public class CoredumpHandlerTest { public void fails_to_get_core_file_if_only_compressed() throws IOException { ContainerPath coredumpDirectory = context.paths().of("/path/to/coredump/proccessing/id-123"); Files.createDirectories(coredumpDirectory); - Files.createFile(coredumpDirectory.resolve("dump_bash.core.431.lz4")); + Files.createFile(coredumpDirectory.resolve("dump_bash.core.431.zstd")); coredumpHandler.findCoredumpFileInProcessingDirectory(coredumpDirectory); } @@ -199,18 +195,12 @@ public class CoredumpHandlerTest { Files.createFile(coredumpDirectory.resolve("dump_bash.core.431")); assertFolderContents(coredumpDirectory, "metadata.json", "dump_bash.core.431"); - terminal.interceptCommand("/usr/bin/lz4 -f /data/vespa/storage/container-123/path/to/coredump/proccessing/id-123/dump_bash.core.431 " + - "/data/vespa/storage/container-123/path/to/coredump/proccessing/id-123/dump_bash.core.431.lz4 2>&1", - commandLine -> { - uncheck(() -> Files.createFile(fileSystem.getPath(commandLine.getArguments().get(3)))); - return new TestChildProcess2(0, ""); - }); coredumpHandler.processAndReportSingleCoredump(context, coredumpDirectory, Map::of); verify(coreCollector, never()).collect(any(), any()); verify(coredumpReporter, times(1)).reportCoredump(eq("id-123"), eq("metadata")); assertFalse(Files.exists(coredumpDirectory)); assertFolderContents(doneCoredumpsPath.resolve("container-123"), "id-123"); - assertFolderContents(doneCoredumpsPath.resolve("container-123").resolve("id-123"), "metadata.json", "dump_bash.core.431.lz4"); + assertFolderContents(doneCoredumpsPath.resolve("container-123").resolve("id-123"), "metadata.json", "dump_bash.core.431.zstd"); } @Test @@ -240,11 +230,6 @@ public class CoredumpHandlerTest { Files.createDirectories(containerCrashPath.pathOnHost()); } - @After - public void teardown() { - terminal.verifyAllCommandsExecuted(); - } - private static void assertFolderContents(Path pathToFolder, String... filenames) { Set<String> expectedContentsOfFolder = Set.of(filenames); Set<String> actualContentsOfFolder; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index e91963ab621..cde7f300f2b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -304,7 +304,7 @@ public final class Node implements Nodelike { /** Returns a copy of this with removable set to the given value */ public Node removable(boolean removable) { - return with(requireAllocation("Cannot set removable").removable(removable)); + return with(requireAllocation("Cannot set removable").removable(removable, false)); } /** Returns a copy of this with the restart generation set to generation */ @@ -385,9 +385,8 @@ public final class Node implements Nodelike { /** Returns a copy of this with allocation set as specified. <code>node.state</code> is *not* changed. */ public Node allocate(ApplicationId owner, ClusterMembership membership, NodeResources requestedResources, Instant at) { - return this - .with(new Allocation(owner, membership, requestedResources, new Generation(0, 0), false)) - .with(history.with(new History.Event(History.Event.Type.reserved, Agent.application, at))); + return this.with(new Allocation(owner, membership, requestedResources, new Generation(0, 0), false)) + .with(history.with(new History.Event(History.Event.Type.reserved, Agent.application, at))); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index 9a110427223..dd4d5aa213f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.node.ClusterId; -import com.yahoo.vespa.hosted.provision.node.Report; import java.util.Comparator; import java.util.EnumSet; @@ -57,7 +56,12 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> { /** Returns the subset of nodes which are removable */ public NodeList removable() { - return matching(node -> node.allocation().isPresent() && node.allocation().get().isRemovable()); + return matching(node -> node.allocation().isPresent() && node.allocation().get().removable()); + } + + /** Returns the subset of nodes which are reusable immediately after removal */ + public NodeList reusable() { + return matching(node -> node.allocation().isPresent() && node.allocation().get().reusable()); } /** Returns the subset of nodes having exactly the given resources */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 9623406767a..c2e66d39861 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -115,7 +115,7 @@ public class Autoscaler { // The cluster is processing recent changes if (clusterNodes.stream().anyMatch(node -> node.status().wantToRetire() || node.allocation().get().membership().retired() || - node.allocation().get().isRemovable())) + node.allocation().get().removable())) return false; // A deployment is ongoing diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index d6ef152dcb2..b3ff0c42547 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -89,7 +89,7 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer { } } - /** Returns the last time application was deployed. Epoch is returned if the application has never been deployed. */ + /** Returns the last time application was activated. Epoch is returned if the application has never been deployed. */ protected final Instant getLastDeployTime(ApplicationId application) { return deployer.lastDeployTime(application).orElse(Instant.EPOCH); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index a461d98697d..9c6eb2199f5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -204,7 +204,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { .collect(Collectors.toMap(Node::hostname, Function.identity()))); nodes.stream() - .filter(node -> node.allocation().isPresent() && !node.status().wantToDeprovision()) + .filter(node -> node.allocation().isPresent()) .flatMap(node -> node.parentHostname().stream()) .distinct() .forEach(hostsByHostname::remove); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 5e703139f41..f32fd225427 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -125,7 +125,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { DefaultTimes(Zone zone, Deployer deployer) { autoscalingInterval = Duration.ofMinutes(15); - dynamicProvisionerInterval = Duration.ofMinutes(5); + dynamicProvisionerInterval = Duration.ofMinutes(3); failedExpirerInterval = Duration.ofMinutes(10); failGrace = Duration.ofMinutes(30); infrastructureProvisionInterval = Duration.ofMinutes(3); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java index 73c9a1ab55a..3143fb5bcfe 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java @@ -14,9 +14,7 @@ import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * Maintenance job which deactivates retired nodes, if given permission by orchestrator, or @@ -53,42 +51,48 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { for (Map.Entry<ApplicationId, NodeList> entry : retiredNodesByApplication.entrySet()) { ApplicationId application = entry.getKey(); NodeList retiredNodes = entry.getValue(); - List<Node> nodesToRemove = retiredNodes.stream().filter(n -> canRemove(n, activeNodes)).collect(Collectors.toList()); - if (nodesToRemove.isEmpty()) continue; - - attempts++; - try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) { - if ( ! deployment.isValid()) continue; - - nodeRepository().nodes().setRemovable(application, nodesToRemove); - boolean success = deployment.activate().isPresent(); - if ( ! success) continue; - String nodeList = nodesToRemove.stream().map(Node::hostname).collect(Collectors.joining(", ")); - log.info("Redeployed " + application + " to deactivate retired nodes: " + nodeList); - successes++; + Map<Removal, NodeList> nodesByRemovalReason = retiredNodes.groupingBy(node -> removalOf(node, activeNodes)); + if (nodesByRemovalReason.isEmpty()) continue; + + for (var kv : nodesByRemovalReason.entrySet()) { + Removal removal = kv.getKey(); + if (removal.equals(Removal.none())) continue; + + NodeList nodes = kv.getValue(); + attempts++; + try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) { + if (!deployment.isValid()) continue; + + nodeRepository().nodes().setRemovable(application, nodes.asList(), removal.isReusable()); + boolean success = deployment.activate().isPresent(); + if (!success) continue; + String nodeList = String.join(", ", nodes.mapToList(Node::hostname)); + log.info("Redeployed " + application + " to deactivate retired nodes: " + nodeList); + successes++; + } } } return attempts == 0 ? 1.0 : ((double)successes / attempts); } /** - * Checks if the node can be removed: - * if the node is a host, it will only be removed if it has no children, - * or all its children are parked or failed. + * Returns the removal action for given node. + * + * If the node is a host, it will only be removed if it has no children, or all its children are parked or failed. + * * Otherwise, a removal is allowed if either of these are true: * - The node has been in state {@link History.Event.Type#retired} for longer than {@link #retiredExpiry} * - Orchestrator allows it */ - private boolean canRemove(Node node, NodeList activeNodes) { + private Removal removalOf(Node node, NodeList activeNodes) { if (node.type().isHost()) { if (nodeRepository().nodes().list().childrenOf(node).asList().stream() .allMatch(child -> child.state() == Node.State.parked || child.state() == Node.State.failed)) { log.info("Allowing removal of " + node + ": host has no non-parked/failed children"); - return true; + return Removal.reusable(); // Hosts have no state that needs to be recoverable } - - return false; + return Removal.none(); } if (node.type().isConfigServerLike()) { @@ -114,24 +118,32 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { // with node states across all config servers. As this would require some work, // we will instead verify here that there are 3 active config servers before // allowing the removal of any config server. - return false; + return Removal.none(); } } else if (node.history().hasEventBefore(History.Event.Type.retired, clock().instant().minus(retiredExpiry))) { log.warning("Node " + node + " has been retired longer than " + retiredExpiry + ": Allowing removal. This may cause data loss"); - return true; + return Removal.recoverable(); } try { nodeRepository().orchestrator().acquirePermissionToRemove(new HostName(node.hostname())); log.info("Node " + node + " has been granted permission to be removed"); - return true; + return Removal.reusable(); // Node is fully retired } catch (UncheckedTimeoutException e) { log.warning("Timed out trying to acquire permission to remove " + node.hostname() + ": " + Exceptions.toMessageString(e)); - return false; + return Removal.none(); } catch (OrchestrationException e) { log.info("Did not get permission to remove retired " + node + ": " + Exceptions.toMessageString(e)); - return false; + return Removal.none(); } } + private record Removal(boolean isRemovable, boolean isReusable) { + + private static Removal recoverable() { return new Removal(true, false); } + private static Removal reusable() { return new Removal(true, true); } + private static Removal none() { return new Removal(false, false); } + + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java index 9c71d621c69..ec1c6cd4b1a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java @@ -27,25 +27,33 @@ public class Allocation { */ private final Generation restartGeneration; - /** This node can (and should) be removed from the cluster on the next deployment */ + /** This node should be removed from the cluster on the next deployment */ private final boolean removable; + /** This node can be reused (dirtied) immediately after being removed from the cluster */ + private final boolean reusable; + private final Optional<NetworkPorts> networkPorts; public Allocation(ApplicationId owner, ClusterMembership clusterMembership, NodeResources requestedResources, Generation restartGeneration, boolean removable) { - this(owner, clusterMembership, requestedResources, restartGeneration, removable, Optional.empty()); + this(owner, clusterMembership, requestedResources, restartGeneration, removable, false, Optional.empty()); } public Allocation(ApplicationId owner, ClusterMembership clusterMembership, NodeResources requestedResources, - Generation restartGeneration, boolean removable, Optional<NetworkPorts> networkPorts) { + Generation restartGeneration, boolean removable, boolean reusable, + Optional<NetworkPorts> networkPorts) { this.owner = owner; this.clusterMembership = clusterMembership; this.requestedResources = requestedResources; this.restartGeneration = restartGeneration; this.removable = removable; + this.reusable = reusable; this.networkPorts = networkPorts; + if (!removable && reusable) { + throw new IllegalArgumentException("Allocation must be removable in order to be reusable"); + } } /** Returns the id of the application this is allocated to */ @@ -65,37 +73,45 @@ public class Allocation { /** Returns a copy of this which is retired */ public Allocation retire() { - return new Allocation(owner, clusterMembership.retire(), requestedResources, restartGeneration, removable, networkPorts); + return new Allocation(owner, clusterMembership.retire(), requestedResources, restartGeneration, removable, reusable, networkPorts); } /** Returns a copy of this which is not retired */ public Allocation unretire() { - return new Allocation(owner, clusterMembership.unretire(), requestedResources, restartGeneration, removable, networkPorts); + return new Allocation(owner, clusterMembership.unretire(), requestedResources, restartGeneration, removable, reusable, networkPorts); } - /** Return whether this node is ready to be removed from the application */ - public boolean isRemovable() { return removable; } + /** Returns whether this node is ready to be removed from the application */ + public boolean removable() { return removable; } + + /** Returns whether this node has fully retired and can be reused immediately */ + public boolean reusable() { + return reusable; + } public Allocation withRequestedResources(NodeResources resources) { - return new Allocation(owner, clusterMembership, resources, restartGeneration, removable, networkPorts); + return new Allocation(owner, clusterMembership, resources, restartGeneration, removable, reusable, networkPorts); } /** Returns a copy of this with the current restart generation set to generation */ public Allocation withRestart(Generation generation) { - return new Allocation(owner, clusterMembership, requestedResources, generation, removable, networkPorts); + return new Allocation(owner, clusterMembership, requestedResources, generation, removable, reusable, networkPorts); } - /** Returns a copy of this allocation where removable is set to the given value */ - public Allocation removable(boolean removable) { - return new Allocation(owner, clusterMembership, requestedResources, restartGeneration, removable, networkPorts); + /** + * Returns a copy of this allocation where removable and reusable are set to the given values. A node which is + * reusable may be moved directly to {@link com.yahoo.vespa.hosted.provision.Node.State#dirty} after removal. + */ + public Allocation removable(boolean removable, boolean reusable) { + return new Allocation(owner, clusterMembership, requestedResources, restartGeneration, removable, reusable, networkPorts); } public Allocation with(ClusterMembership newMembership) { - return new Allocation(owner, newMembership, requestedResources, restartGeneration, removable, networkPorts); + return new Allocation(owner, newMembership, requestedResources, restartGeneration, removable, reusable, networkPorts); } public Allocation withNetworkPorts(NetworkPorts ports) { - return new Allocation(owner, clusterMembership, requestedResources, restartGeneration, removable, Optional.of(ports)); + return new Allocation(owner, clusterMembership, requestedResources, restartGeneration, removable, reusable, Optional.of(ports)); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 11d3e03e494..d750a3ef737 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -226,13 +226,14 @@ public class Nodes { * Sets a list of nodes to have their allocation removable (active to inactive) in the node repository. * * @param application the application the nodes belong to - * @param nodes the nodes to make removable. These nodes MUST be in the active state. + * @param nodes the nodes to make removable. These nodes MUST be in the active state + * @param reusable move the node directly to {@link Node.State#dirty} after removal */ - public void setRemovable(ApplicationId application, List<Node> nodes) { + public void setRemovable(ApplicationId application, List<Node> nodes, boolean reusable) { try (Mutex lock = lock(application)) { List<Node> removableNodes = nodes.stream() - .map(node -> node.with(node.allocation().get().removable(true))) - .collect(Collectors.toList()); + .map(node -> node.with(node.allocation().get().removable(true, reusable))) + .toList(); write(removableNodes, lock); } } @@ -247,9 +248,12 @@ public class Nodes { var stateless = NodeList.copyOf(nodes).stateless(); var stateful = NodeList.copyOf(nodes).stateful(); + var statefulToInactive = stateful.not().reusable(); + var statefulToDirty = stateful.reusable(); List<Node> written = new ArrayList<>(); written.addAll(deallocate(stateless.asList(), Agent.application, "Deactivated by application", transaction.nested())); - written.addAll(db.writeTo(Node.State.inactive, stateful.asList(), Agent.application, Optional.empty(), transaction.nested())); + written.addAll(deallocate(statefulToDirty.asList(), Agent.application, "Deactivated by application (recycled)", transaction.nested())); + written.addAll(db.writeTo(Node.State.inactive, statefulToInactive.asList(), Agent.application, Optional.empty(), transaction.nested())); return written; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 083b707b3c5..24c52f7b2e0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -115,6 +115,7 @@ public class NodeSerializer { private static final String restartGenerationKey = "restartGeneration"; private static final String currentRestartGenerationKey = "currentRestartGeneration"; private static final String removableKey = "removable"; + private static final String reusableKey = "reusable"; // Saved as part of allocation instead of serviceId, since serviceId serialized form is not easily extendable. private static final String wantedVespaVersionKey = "wantedVespaVersion"; private static final String wantedContainerImageRepoKey = "wantedDockerImageRepo"; @@ -217,7 +218,8 @@ public class NodeSerializer { object.setString(serviceIdKey, allocation.membership().stringValue()); object.setLong(restartGenerationKey, allocation.restartGeneration().wanted()); object.setLong(currentRestartGenerationKey, allocation.restartGeneration().current()); - object.setBool(removableKey, allocation.isRemovable()); + object.setBool(removableKey, allocation.removable()); + object.setBool(reusableKey, allocation.reusable()); object.setString(wantedVespaVersionKey, allocation.membership().cluster().vespaVersion().toString()); allocation.membership().cluster().dockerImageRepo().ifPresent(repo -> object.setString(wantedContainerImageRepoKey, repo.untagged())); allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray(networkPortsKey))); @@ -330,6 +332,7 @@ public class NodeSerializer { .orElse(assignedResources), generationFromSlime(object, restartGenerationKey, currentRestartGenerationKey), object.field(removableKey).asBool(), + object.field(reusableKey).asBool(), NetworkPortsSerializer.fromSlime(object.field(networkPortsKey)))); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index 3da0506f2e1..5d73c4929af 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -88,8 +88,7 @@ class Activator { validateParentHosts(application, allNodes, reserved); NodeList activeToRemove = oldActive.matching(node -> ! hostnames.contains(node.hostname())); - activeToRemove = NodeList.copyOf(activeToRemove.mapToList(Node::unretire)); // only active nodes can be retired. TODO: Move this line to deactivate - deactivate(activeToRemove, transaction); // TODO: Pass activation time in this call and next line + remove(activeToRemove, transaction); // TODO: Pass activation time in this call and next line nodeRepository.nodes().activate(newActive.asList(), transaction.nested()); // activate also continued active to update node state rememberResourceChange(transaction, generation, activationTime, @@ -99,9 +98,10 @@ class Activator { return newActive; } - private void deactivate(NodeList toDeactivate, ApplicationTransaction transaction) { - nodeRepository.nodes().deactivate(toDeactivate.not().failing().asList(), transaction); - nodeRepository.nodes().fail(toDeactivate.failing().asList(), transaction); + private void remove(NodeList nodes, ApplicationTransaction transaction) { + nodes = NodeList.copyOf(nodes.mapToList(Node::unretire)); // clear retire flag when moving to non-active state + nodeRepository.nodes().deactivate(nodes.not().failing().asList(), transaction); + nodeRepository.nodes().fail(nodes.failing().asList(), transaction); } private void rememberResourceChange(ApplicationTransaction transaction, long generation, Instant at, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index a26df62be27..9557354725b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -112,7 +112,7 @@ class NodeAllocation { if ( ! allocation.owner().equals(application)) continue; // wrong application if ( ! membership.cluster().satisfies(cluster)) continue; // wrong cluster id/type if ((! candidate.isSurplus || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it - if ( candidate.state() == Node.State.active && allocation.isRemovable()) continue; // don't accept; causes removal + if ( candidate.state() == Node.State.active && allocation.removable()) continue; // don't accept; causes removal if ( candidate.state() == Node.State.active && candidate.wantToFail()) continue; // don't accept; causes failing if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java index 56f4863c881..62c96af7629 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java @@ -105,21 +105,21 @@ public class NodeRepoStatsTest { var app3Stats = stats.applicationStats().get(1); assertEquals(app1, app1Stats.id()); - assertEquals(2.940, app1Stats.cost(), delta); - assertEquals(0.702, app1Stats.utilizedCost(), delta); - assertEquals(2.238, app1Stats.unutilizedCost(), delta); + assertEquals(3.6400, app1Stats.cost(), delta); + assertEquals(0.8676, app1Stats.utilizedCost(), delta); + assertEquals(2.7724, app1Stats.unutilizedCost(), delta); assertLoad(new Load(0.2571, 0.2314, 0.2057), app1Stats.load()); assertEquals(app2, app2Stats.id()); - assertEquals(1.680, app2Stats.cost(), delta); - assertEquals(0.624, app2Stats.utilizedCost(), delta); - assertEquals(1.056, app2Stats.unutilizedCost(), delta); + assertEquals(2.0799, app2Stats.cost(), delta); + assertEquals(0.7712, app2Stats.utilizedCost(), delta); + assertEquals(1.3087, app2Stats.unutilizedCost(), delta); assertLoad(new Load(.40, 0.36, 0.32), app2Stats.load()); assertEquals(app3, app3Stats.id()); - assertEquals(2.100, app3Stats.cost(), delta); - assertEquals(0.975, app3Stats.utilizedCost(), delta); - assertEquals(1.125, app3Stats.unutilizedCost(), delta); + assertEquals(2.6000, app3Stats.cost(), delta); + assertEquals(1.2049, app3Stats.utilizedCost(), delta); + assertEquals(1.3950, app3Stats.unutilizedCost(), delta); assertLoad(new Load(0.5, 0.45, 0.40), app3Stats.load()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 0c164328086..94b938fc886 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -124,7 +124,7 @@ class AutoscalingTester { try (Mutex lock = nodeRepository().nodes().lock(application)){ for (Node node : nodeRepository().nodes().list(Node.State.active).owner(application)) { if (node.allocation().get().membership().retired()) - nodeRepository().nodes().write(node.with(node.allocation().get().removable(true)), lock); + nodeRepository().nodes().write(node.with(node.allocation().get().removable(true, true)), lock); } } deploy(application, cluster, resources); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 451bcfb0b78..905fdc57813 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -457,7 +457,7 @@ public class DynamicProvisioningMaintainerTest { // Initial config server hosts are provisioned manually List<Node> provisionedHosts = tester.makeReadyNodes(3, "default", hostType, 1).stream() .sorted(Comparator.comparing(Node::hostname)) - .collect(Collectors.toList()); + .toList(); tester.prepareAndActivateInfraApplication(hostApp, hostType); // Provision config servers @@ -485,18 +485,14 @@ public class DynamicProvisioningMaintainerTest { assertTrue("Redeployment retires node", nodeToRemove.get().allocation().get().membership().retired()); // Config server becomes removable (done by RetiredExpirer in a real system) and redeployment moves it - // to inactive - tester.nodeRepository().nodes().setRemovable(configSrvApp, List.of(nodeToRemove.get())); - tester.prepareAndActivateInfraApplication(configSrvApp, hostType.childNodeType()); - assertEquals("Node moves to inactive", Node.State.inactive, nodeToRemove.get().state()); - - // Node is parked (done by InactiveExpirer and host-admin in a real system) + // to parked int removedIndex = nodeToRemove.get().allocation().get().membership().index(); - Node parkedConfigServer = tester.nodeRepository().nodes().deallocate(nodeToRemove.get(), Agent.system, getClass().getSimpleName()); - assertSame("Node moves to parked", Node.State.parked, parkedConfigServer.state()); + tester.nodeRepository().nodes().setRemovable(configSrvApp, List.of(nodeToRemove.get()), true); + tester.prepareAndActivateInfraApplication(configSrvApp, hostType.childNodeType()); + assertSame("Node moves to expected state", Node.State.parked, nodeToRemove.get().state()); assertEquals(2, tester.nodeRepository().nodes().list().nodeType(hostType.childNodeType()).state(Node.State.active).size()); - // ... same for host + // Host is parked (done by DynamicProvisioningMaintainer in a real system) tester.nodeRepository().nodes().deallocate(hostToRemove.get(), Agent.system, getClass().getSimpleName()); assertSame("Host moves to parked", Node.State.parked, hostToRemove.get().state()); @@ -574,25 +570,6 @@ public class DynamicProvisioningMaintainerTest { assertEquals(2, provisioningTester.activate(applicationId, prepared).size()); } - @Test - public void deprovision_parked_node_with_allocation() { - var tester = new DynamicProvisioningTester(); - tester.hostProvisioner.with(Behaviour.failProvisioning); - Node host4 = tester.addNode("host4", Optional.empty(), NodeType.host, Node.State.parked); - Node host41 = tester.addNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.parked, DynamicProvisioningTester.tenantApp); - tester.nodeRepository.nodes().deprovision("host4", Agent.operator, Instant.now()); - - assertEquals(Optional.of(true), tester.nodeRepository.nodes().node("host4").map(n -> n.status().wantToDeprovision())); - assertEquals(Optional.of(Node.State.parked), tester.nodeRepository.nodes().node("host4").map(Node::state)); - assertEquals(Optional.of(true), tester.nodeRepository.nodes().node("host4-1").map(n -> n.status().wantToDeprovision())); - assertEquals(Optional.of(Node.State.parked), tester.nodeRepository.nodes().node("host4-1").map(Node::state)); - - tester.maintainer.maintain(); - - assertEquals(Optional.empty(), tester.nodeRepository.nodes().node("host4")); - assertEquals(Optional.empty(), tester.nodeRepository.nodes().node("host4-1")); - } - private void assertCfghost3IsActive(DynamicProvisioningTester tester) { assertEquals(5, tester.nodeRepository.nodes().list(Node.State.active).size()); assertEquals(3, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.confighost).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 1237ede5345..bf98a875e42 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -112,7 +112,7 @@ public class RetiredExpirerTest { } @Test - public void ensure_early_inactivation() throws OrchestrationException { + public void retired_nodes_are_dellocated() throws OrchestrationException { tester.makeReadyNodes(7, nodeResources); tester.makeReadyHosts(4, hostResources); @@ -148,27 +148,35 @@ public class RetiredExpirerTest { RetiredExpirer retiredExpirer = createRetiredExpirer(deployer); retiredExpirer.run(); assertEquals(5, nodeRepository.nodes().list(Node.State.active).owner(applicationId).size()); - assertEquals(2, nodeRepository.nodes().list(Node.State.inactive).owner(applicationId).size()); + assertEquals(2, nodeRepository.nodes().list(Node.State.dirty).owner(applicationId).size()); assertEquals(1, deployer.redeployments); verify(orchestrator, times(4)).acquirePermissionToRemove(any()); // Running it again has no effect retiredExpirer.run(); assertEquals(5, nodeRepository.nodes().list(Node.State.active).owner(applicationId).size()); - assertEquals(2, nodeRepository.nodes().list(Node.State.inactive).owner(applicationId).size()); + assertEquals(2, nodeRepository.nodes().list(Node.State.dirty).owner(applicationId).size()); assertEquals(1, deployer.redeployments); verify(orchestrator, times(6)).acquirePermissionToRemove(any()); + // Running it again deactivates nodes that have exceeded max retirement period clock.advance(RETIRED_EXPIRATION.plusMinutes(1)); retiredExpirer.run(); assertEquals(3, nodeRepository.nodes().list(Node.State.active).owner(applicationId).size()); - assertEquals(4, nodeRepository.nodes().list(Node.State.inactive).owner(applicationId).size()); + assertEquals(2, nodeRepository.nodes().list(Node.State.dirty).owner(applicationId).size()); + assertEquals(2, nodeRepository.nodes().list(Node.State.inactive).owner(applicationId).size()); assertEquals(2, deployer.redeployments); verify(orchestrator, times(6)).acquirePermissionToRemove(any()); - // inactivated nodes are not retired - for (Node node : nodeRepository.nodes().list(Node.State.inactive).owner(applicationId)) + // Removed nodes are not retired + for (Node node : nodeRepository.nodes().list(Node.State.inactive, Node.State.dirty).owner(applicationId)) { + if (node.state() == Node.State.inactive) { + assertFalse(node + " node is reusable", node.allocation().get().reusable()); + } else { + assertTrue(node + " is reusable", node.allocation().get().reusable()); + } assertFalse(node.allocation().get().membership().retired()); + } } @Test @@ -207,19 +215,19 @@ public class RetiredExpirerTest { assertTrue(activeConfigServerHostnames.contains(retiredNode.hostname())); activeConfigServerHostnames.remove(retiredNode.hostname()); assertEquals(activeConfigServerHostnames, configServerHostnames(duperModel)); - assertEquals(1, tester.nodeRepository().nodes().list(Node.State.inactive).nodeType(NodeType.config).size()); + assertEquals(1, tester.nodeRepository().nodes().list(Node.State.parked).nodeType(NodeType.config).size()); assertEquals(2, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.config).size()); - // no changes while 1 cfg is inactive + // no changes while 1 cfg is dirty retiredExpirer.run(); assertEquals(activeConfigServerHostnames, configServerHostnames(duperModel)); - assertEquals(1, tester.nodeRepository().nodes().list(Node.State.inactive).nodeType(NodeType.config).size()); + NodeList parked = tester.nodeRepository().nodes().list(Node.State.parked).nodeType(NodeType.config); + assertEquals(1, parked.size()); assertEquals(2, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.config).size()); - // The node will eventually expire from inactive, and be removed by DynamicProvisioningMaintainer - // (depending on its host), and these events should not affect the 2 active config servers. - nodeRepository.nodes().deallocate(retiredNode, Agent.InactiveExpirer, "expired"); - retiredNode = tester.nodeRepository().nodes().list(Node.State.parked).nodeType(NodeType.config).asList().get(0); + // Node is removed by DynamicProvisioningMaintainer (depending on its host), and these events should not affect + // the 2 active config servers. + retiredNode = parked.first().get(); nodeRepository.nodes().removeRecursively(retiredNode, true); infraDeployer.activateAllSupportedInfraApplications(true); retiredExpirer.run(); @@ -228,7 +236,7 @@ public class RetiredExpirerTest { assertEquals(2, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.config).size()); // Provision and ready new config server - MockNameResolver nameResolver = (MockNameResolver)tester.nodeRepository().nameResolver(); + MockNameResolver nameResolver = (MockNameResolver) tester.nodeRepository().nameResolver(); String ipv4 = "127.0.1.4"; nameResolver.addRecord(retiredNode.hostname(), ipv4); Node node = Node.create(retiredNode.hostname(), new IP.Config(Set.of(ipv4), Set.of()), retiredNode.hostname(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 26be6d95336..97a8ac0d655 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -469,7 +469,7 @@ public class OsVersionsTest { deployApplication(application); List<Node> retired = tester.nodeRepository().nodes().list().owner(application).retired().asList(); assertFalse("At least one node is retired", retired.isEmpty()); - tester.nodeRepository().nodes().setRemovable(application, retired); + tester.nodeRepository().nodes().setRemovable(application, retired, false); // Redeploy to deactivate removable nodes and allocate new ones deployApplication(application); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index bc7d3104a2f..f22abf4c9e3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -114,7 +114,7 @@ public class NodeSerializerTest { assertEquals(node.allocation().get().owner(), copy.allocation().get().owner()); assertEquals(node.allocation().get().membership(), copy.allocation().get().membership()); assertEquals(node.allocation().get().requestedResources(), copy.allocation().get().requestedResources()); - assertEquals(node.allocation().get().isRemovable(), copy.allocation().get().isRemovable()); + assertEquals(node.allocation().get().removable(), copy.allocation().get().removable()); assertEquals(4, copy.history().events().size()); assertEquals(5, copy.history().log().size()); assertEquals(Instant.ofEpochMilli(1), copy.history().event(History.Event.Type.reserved).get().at()); @@ -167,7 +167,7 @@ public class NodeSerializerTest { assertEquals(4, node.allocation().get().restartGeneration().current()); assertEquals(List.of(History.Event.Type.provisioned, History.Event.Type.reserved), node.history().events().stream().map(History.Event::type).collect(Collectors.toList())); - assertTrue(node.allocation().get().isRemovable()); + assertTrue(node.allocation().get().removable()); assertEquals(NodeType.tenant, node.type()); } @@ -193,9 +193,10 @@ public class NodeSerializerTest { (copy.history().event(History.Event.Type.retired).get()).agent()); assertTrue(copy.allocation().get().membership().retired()); - Node removable = copy.with(node.allocation().get().removable(true)); + Node removable = copy.with(node.allocation().get().removable(true, true)); Node removableCopy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(removable)); - assertTrue(removableCopy.allocation().get().isRemovable()); + assertTrue(removableCopy.allocation().get().removable()); + assertTrue(removableCopy.allocation().get().reusable()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java index 4f2360590c4..5149b234ccf 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java @@ -125,7 +125,7 @@ public class InfraDeployerImplTest { addNode(5, Node.State.dirty, Optional.empty()); addNode(6, Node.State.ready, Optional.empty()); Node node7 = addNode(7, Node.State.active, Optional.of(target)); - nodeRepository.nodes().setRemovable(application.getApplicationId(), List.of(node7)); + nodeRepository.nodes().setRemovable(application.getApplicationId(), List.of(node7), false); infraDeployer.getDeployment(application.getApplicationId()).orElseThrow().activate(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index 798ed1e45a7..a1c55833862 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -75,7 +75,7 @@ public class VirtualNodeProvisioningTest { // Go down to 3 nodes in container cluster List<HostSpec> containerHosts2 = tester.prepare(applicationId, containerClusterSpec, containerNodeCount - 1, groups, resources1); - tester.activate(applicationId, containerHosts2); + tester.activate(applicationId, concat(containerHosts2, contentHosts)); NodeList nodes2 = tester.getNodes(applicationId, Node.State.active); assertDistinctParentHosts(nodes2, ClusterSpec.Type.container, containerNodeCount - 1); @@ -637,7 +637,7 @@ public class VirtualNodeProvisioningTest { tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, r))); // Deactivate any retired nodes - usually done by the RetiredExpirer - tester.nodeRepository().nodes().setRemovable(app1, tester.getNodes(app1).retired().asList()); + tester.nodeRepository().nodes().setRemovable(app1, tester.getNodes(app1).retired().asList(), false); tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, r))); if (expectedReuse) { @@ -691,7 +691,7 @@ public class VirtualNodeProvisioningTest { } private static <T> List<T> concat(List<T> list1, List<T> list2) { - return Stream.concat(list1.stream(), list2.stream()).collect(Collectors.toList()); + return Stream.concat(list1.stream(), list2.stream()).toList(); } } diff --git a/parent/pom.xml b/parent/pom.xml index c9c77f0a30e..a0925faa7ba 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -980,12 +980,12 @@ <apache.httpclient.version>4.5.13</apache.httpclient.version> <apache.httpcore.version>4.4.13</apache.httpcore.version> <apache.httpclient5.version>5.1.3</apache.httpclient5.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> - <asm.version>9.2</asm.version> + <asm.version>9.3</asm.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> <athenz.version>1.10.54</athenz.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> <jjwt.version>0.11.2</jjwt.version> - <aws.sdk.version>1.11.974</aws.sdk.version> + <aws.sdk.version>1.12.246</aws.sdk.version> <!-- Athenz END --> <!-- WARNING: If you change curator version, you also need to update @@ -1004,7 +1004,7 @@ <findbugs.version>3.0.2</findbugs.version> <!-- Should be kept in sync with guava --> <gson.version>2.8.9</gson.version> <hdrhistogram.version>2.1.12</hdrhistogram.version> - <jetty.version>9.4.46.v20220331</jetty.version> + <jetty.version>9.4.48.v20220622</jetty.version> <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> <jna.version>5.11.0</jna.version> <junit.version>5.8.1</junit.version> diff --git a/screwdriver/release-container-image.sh b/screwdriver/release-container-image.sh index 30287bc5b52..3ae2a7feb1e 100755 --- a/screwdriver/release-container-image.sh +++ b/screwdriver/release-container-image.sh @@ -26,7 +26,7 @@ cd $BUILD_DIR git clone --depth 1 https://github.com/vespa-engine/docker-image cd docker-image -docker build --build-arg VESPA_VERSION=$VESPA_VERSION --file Dockerfile.stream8 \ +docker build --build-arg VESPA_VERSION=$VESPA_VERSION --file Dockerfile \ --tag docker.io/vespaengine/vespa:$VESPA_VERSION --tag docker.io/vespaengine/vespa:latest \ --tag ghcr.io/vespa-engine/vespa:$VESPA_VERSION --tag ghcr.io/vespa-engine/vespa:latest . diff --git a/screwdriver/release-java-artifacts.sh b/screwdriver/release-java-artifacts.sh index 8d80bb45578..0c8ae0e4eb1 100755 --- a/screwdriver/release-java-artifacts.sh +++ b/screwdriver/release-java-artifacts.sh @@ -52,6 +52,10 @@ for MODULE in $(comm -2 -3 \ echo "No javadoc available for module" > $MODULE/src/main/javadoc/README done +# Workaround new module without java code and no javadoc +mkdir -p container-spifly/src/main/javadoc +echo "No javadoc available for module" > container-spifly/src/main/javadoc/README + # Workaround for broken nexus-staging-maven-plugin instead of swapping JDK export MAVEN_OPTS="--add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.desktop/java.awt.font=ALL-UNNAMED" export VESPA_MAVEN_EXTRA_OPTS="--show-version --batch-mode" diff --git a/screwdriver/release-rpms.sh b/screwdriver/release-rpms.sh index 256bfd8c11a..c6206fd662e 100755 --- a/screwdriver/release-rpms.sh +++ b/screwdriver/release-rpms.sh @@ -12,7 +12,7 @@ fi readonly VESPA_RELEASE="$1" readonly VESPA_REF="$2" -VESPA_RPM=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-7-x86_64 --repoid=vespa -q vespa | tail -1 | cut -d: -f2 | cut -d- -f1) +VESPA_RPM=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/centos-stream-8-x86_64 --repoid=vespa -q vespa | tail -1 | cut -d: -f2 | cut -d- -f1) echo "Latest RPM on Copr: $VESPA_RPM" if [ "$VESPA_RELEASE" == "$VESPA_RPM" ]; then @@ -32,8 +32,8 @@ cd vespa dist/release-vespa-rpm.sh $VESPA_RELEASE $VESPA_REF while [ "$VESPA_RELEASE" != "$VESPA_RPM" ]; do - dnf clean --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-7-x86_64 --repoid=vespa metadata - VESPA_RPM=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-7-x86_64 --repoid=vespa -q vespa | tail -1 | cut -d: -f2 | cut -d- -f1) + dnf clean --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/centos-stream-8-x86_64 --repoid=vespa metadata + VESPA_RPM=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/centos-stream-8-x86_64 --repoid=vespa -q vespa | tail -1 | cut -d: -f2 | cut -d- -f1) echo "RPM: $VESPA_RPM" sleep 150 done diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index b489a07bba2..a9c18d4b8a9 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -379,7 +379,7 @@ public: } void clear() override { - FastOS_FileInterface::EmptyAndRemoveDirectory(_baseDir.c_str()); + std::filesystem::remove_all(std::filesystem::path(_baseDir)); } bool hasPersistence() const override { return true; } diff --git a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp index 2d7e242b9fb..e367df3abbd 100644 --- a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp +++ b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp @@ -159,7 +159,7 @@ check(const AttributePtr &vec, uint32_t docId, const std::vector<T> &values) uint32_t sz = vec->getValueCount(docId); if (!EXPECT_EQUAL(sz, values.size())) return false; std::vector<T> buf(sz); - uint32_t asz = vec->get(docId, &buf[0], sz); + uint32_t asz = vec->get(docId, buf.data(), sz); if (!EXPECT_EQUAL(sz, asz)) return false; std::vector<T> wanted(values.begin(), values.end()); if (vec->hasWeightedSetType()) { diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index a0400661b4e..49fd82c3a36 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -42,6 +42,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/size_literals.h> +#include <filesystem> #include <regex> #include <vespa/log/log.h> @@ -82,11 +83,11 @@ public: DirMaker(const vespalib::string & dir) : _dir(dir) { - FastOS_File::MakeDirectory(dir.c_str()); + std::filesystem::create_directory(std::filesystem::path(dir)); } ~DirMaker() { - FastOS_File::EmptyAndRemoveDirectory(_dir.c_str()); + std::filesystem::remove_all(std::filesystem::path(_dir)); } private: vespalib::string _dir; @@ -180,7 +181,7 @@ public: vespalib::ThreadStackExecutor _summaryExecutor; MockSharedThreadingService _shared_service; TransLogServer _tls; - bool _mkdirOk; + bool _made_dir; matching::QueryLimiter _queryLimiter; DummyWireService _dummy; ::config::DirSpec _spec; @@ -199,7 +200,7 @@ public: _summaryExecutor(8, 128_Ki), _shared_service(_summaryExecutor, _summaryExecutor), _tls(_shared_service.transport(), "tmp", 9013, ".", _fileHeaderContext), - _mkdirOk(FastOS_File::MakeDirectory("tmpdb")), + _made_dir(std::filesystem::create_directory(std::filesystem::path("tmpdb"))), _queryLimiter(), _dummy(), _spec(TEST_PATH("")), @@ -212,7 +213,7 @@ public: _aw(), _sa() { - assert(_mkdirOk); + (void) _made_dir; auto b = std::make_shared<BootstrapConfig>(1, _documenttypesConfig, _repo, std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), @@ -220,9 +221,7 @@ public: _tuneFileDocumentDB, _hwInfo); _configMgr.forwardConfig(b); _configMgr.nextGeneration(_shared_service.transport(), 0ms); - if (! FastOS_File::MakeDirectory((std::string("tmpdb/") + docTypeName).c_str())) { - LOG_ABORT("should not be reached"); - } + std::filesystem::create_directory(std::filesystem::path(std::string("tmpdb/") + docTypeName)); _ddb = DocumentDB::create("tmpdb", _configMgr.getConfig(), "tcp/localhost:9013", _queryLimiter, DocTypeName(docTypeName), makeBucketSpace(), *b->getProtonConfigSP(), *this, _shared_service, _tls, _dummy, _fileHeaderContext, @@ -239,8 +238,8 @@ public: _sa.reset(); _aw.reset(); _ddb.reset(); - FastOS_File::EmptyAndRemoveDirectory("tmp"); - FastOS_File::EmptyAndRemoveDirectory("tmpdb"); + std::filesystem::remove_all(std::filesystem::path("tmp")); + std::filesystem::remove_all(std::filesystem::path("tmpdb")); } void diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp index 5c4d7aca41d..6c1821dc00e 100644 --- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp @@ -232,9 +232,9 @@ TEST("require that serialize/deserialize works for CompactLidSpaceOperation") op.serialize(stream); } { - const document::DocumentTypeRepo *repo = NULL; + const document::DocumentTypeRepo repo; CompactLidSpaceOperation op; - op.deserialize(stream, *repo); + op.deserialize(stream, repo); EXPECT_EQUAL(FeedOperation::COMPACT_LID_SPACE, op.getType()); EXPECT_EQUAL(2u, op.getSubDbId()); EXPECT_EQUAL(99u, op.getLidLimit()); diff --git a/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp b/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp index c814bdf3f37..35583ea46da 100644 --- a/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp +++ b/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/fastos/file.h> #include <algorithm> +#include <filesystem> #include <vespa/log/log.h> LOG_SETUP("diskindexcleaner_test"); @@ -30,7 +31,7 @@ public: const string index_dir = "diskindexcleaner_test_data"; void removeTestData() { - FastOS_FileInterface::EmptyAndRemoveDirectory(index_dir.c_str()); + std::filesystem::remove_all(std::filesystem::path(index_dir)); } int @@ -52,9 +53,9 @@ Test::Main() } void createIndex(const string &name) { - FastOS_FileInterface::MakeDirIfNotPresentOrExit(index_dir.c_str()); + std::filesystem::create_directory(std::filesystem::path(index_dir)); const string dir_name = index_dir + "/" + name; - FastOS_FileInterface::MakeDirIfNotPresentOrExit(dir_name.c_str()); + std::filesystem::create_directory(std::filesystem::path(dir_name)); const string serial_file = dir_name + "/serial.dat"; FastOS_File file(serial_file.c_str()); file.OpenWriteOnlyTruncate(); diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp index 4a84e3972ad..850f8a8f0d1 100644 --- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp +++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp @@ -20,6 +20,7 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/fastos/file.h> +#include <filesystem> #include <set> using document::Document; @@ -134,7 +135,7 @@ Schema getSchema() { } void Test::setUp() { - FastOS_FileInterface::EmptyAndRemoveDirectory(base_dir.c_str()); + std::filesystem::remove_all(std::filesystem::path(base_dir)); _fusion_runner.reset(new FusionRunner(base_dir, getSchema(), TuneFileAttributes(), _fileHeaderContext)); @@ -144,7 +145,7 @@ void Test::setUp() { } void Test::tearDown() { - FastOS_FileInterface::EmptyAndRemoveDirectory(base_dir.c_str()); + std::filesystem::remove_all(std::filesystem::path(base_dir)); _selector.reset(0); } @@ -167,7 +168,7 @@ void addDocument(DocBuilder & doc_builder, MemoryIndex &index, ISourceSelector & } void Test::createIndex(const string &dir, uint32_t id, bool fusion) { - FastOS_FileInterface::MakeDirIfNotPresentOrExit(dir.c_str()); + std::filesystem::create_directory(std::filesystem::path(dir)); vespalib::asciistream ost; if (fusion) { ost << dir << "/index.fusion." << id; diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 0bc11e64df8..b427daa4ad1 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -85,7 +85,7 @@ Schema getSchema() { } void removeTestData() { - FastOS_FileInterface::EmptyAndRemoveDirectory(index_dir.c_str()); + std::filesystem::remove_all(std::filesystem::path(index_dir)); } Document::UP buildDocument(DocBuilder &doc_builder, int id, diff --git a/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp b/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp index 4c2afe32ae8..0cf7aea6c00 100644 --- a/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp +++ b/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp @@ -32,7 +32,7 @@ std::tuple<size_t,Hits,RangePair> second_phase(MatchLoopCommunicator &com, const for (size_t i = 0; i < hits.size(); ++i) { refs.push_back(i); } - auto my_work = com.get_second_phase_work(SortedHitSequence(&hits[0], &refs[0], refs.size()), thread_id); + auto my_work = com.get_second_phase_work(SortedHitSequence(hits.data(), refs.data(), refs.size()), thread_id); // the DocumentScorer used by the match thread will sort on docid here to ensure increasing seek order, this is not needed here size_t work_size = my_work.size(); for (auto &[hit, tag]: my_work) { diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index ffd9db2f58b..5829666c41b 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -483,6 +483,11 @@ hwinfo.cpu.cores int default = 0 restart ## See shared_threading_service_config.cpp for details on how the thread pool sizes are calculated. feeding.concurrency double default = 0.2 restart +## A number between 0.0 and 1.0 telling how nice the feed and background threads shall be. +## A value of 0.0, which is default, means 'not any nicer than anyone else'. +## The scale from 0.0 to 1.0 is not linear. It is OS specific. +feeding.niceness double default = 0.0 restart + ## Maximum number of pending tasks for the master thread in each document db. ## ## This limit is only considered when executing tasks for handling external feed operations. diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp index f0e7cad5758..25d6e3c36a7 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp @@ -5,7 +5,6 @@ #include "attribute_directory.h" #include <vespa/searchlib/attribute/attributefilesavetarget.h> #include <vespa/searchlib/attribute/attributesaver.h> -#include <vespa/searchlib/util/dirtraverse.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/searchlib/common/serialnumfileheadercontext.h> diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index 74e135f021d..2b7f5509acb 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -41,7 +41,6 @@ Memory DOCSUM("docsum"); Memory ERRORS("errors"); Memory TYPE("type"); Memory MESSAGE("message"); -Memory DETAILS("details"); Memory TIMEOUT("timeout"); } @@ -51,14 +50,10 @@ DocsumContext::initState() { const DocsumRequest & req = _request; _docsumState._args.initFromDocsumRequest(req); - _docsumState._docsumcnt = req.hits.size(); - - _docsumState._docsumbuf = (_docsumState._docsumcnt > 0) - ? (uint32_t*)malloc(sizeof(uint32_t) * _docsumState._docsumcnt) - : nullptr; - - for (uint32_t i = 0; i < _docsumState._docsumcnt; i++) { - _docsumState._docsumbuf[i] = req.hits[i].docid; + _docsumState._docsumbuf.clear(); + _docsumState._docsumbuf.reserve(req.hits.size()); + for (uint32_t i = 0; i < req.hits.size(); i++) { + _docsumState._docsumbuf.push_back(req.hits[i].docid); } } @@ -77,7 +72,7 @@ vespalib::Slime::UP DocsumContext::createSlimeReply() { _docsumWriter.InitState(_attrMgr, &_docsumState); - const size_t estimatedChunkSize(std::min(0x200000ul, _docsumState._docsumcnt*0x400ul)); + const size_t estimatedChunkSize(std::min(0x200000ul, _docsumState._docsumbuf.size()*0x400ul)); vespalib::Slime::UP response(std::make_unique<vespalib::Slime>(makeSlimeParams(estimatedChunkSize))); Cursor & root = response->setObject(); Cursor & array = root.setArray(DOCSUMS); @@ -85,17 +80,18 @@ DocsumContext::createSlimeReply() IDocsumWriter::ResolveClassInfo rci = _docsumWriter.resolveClassInfo(_docsumState._args.getResultClassName(), _docsumStore.getSummaryClassId()); _docsumState._omit_summary_features = rci.outputClass->omit_summary_features(); - uint32_t i(0); - for (i = 0; (i < _docsumState._docsumcnt) && !_request.expired(); ++i) { - uint32_t docId = _docsumState._docsumbuf[i]; - Cursor & docSumC = array.addObject(); + uint32_t num_ok(0); + for (uint32_t docId : _docsumState._docsumbuf) { + if (_request.expired() ) { break; } + Cursor &docSumC = array.addObject(); ObjectSymbolInserter inserter(docSumC, docsumSym); if ((docId != search::endDocId) && !rci.mustSkip) { _docsumWriter.insertDocsum(rci, docId, &_docsumState, &_docsumStore, *response, inserter); } + num_ok++; } - if (i != _docsumState._docsumcnt) { - const uint32_t numTimedOut = _docsumState._docsumcnt - i; + if (num_ok != _docsumState._docsumbuf.size()) { + const uint32_t numTimedOut = _docsumState._docsumbuf.size() - num_ok; Cursor & errors = root.setArray(ERRORS); Cursor & timeout = errors.addObject(); timeout.setString(TYPE, TIMEOUT); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp index 53535cddcdf..7786ead49b2 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp @@ -8,7 +8,6 @@ #include <vespa/searchlib/attribute/attributefilesavetarget.h> #include <vespa/searchlib/attribute/attributememorysavetarget.h> #include <vespa/searchlib/attribute/attributesaver.h> -#include <vespa/searchlib/util/dirtraverse.h> #include <vespa/searchlib/common/serialnumfileheadercontext.h> #include <vespa/searchlib/util/filekit.h> #include <filesystem> diff --git a/searchcore/src/vespa/searchcore/proton/matching/document_scorer.cpp b/searchcore/src/vespa/searchcore/proton/matching/document_scorer.cpp index 76c9781d7a5..e955b24cc7f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/document_scorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/document_scorer.cpp @@ -34,8 +34,12 @@ DocumentScorer::DocumentScorer(RankProgram &rankProgram, void DocumentScorer::score(TaggedHits &hits) { + if (hits.empty()) { + return; + } auto sort_on_docid = [](const TaggedHit &a, const TaggedHit &b){ return (a.first.first < b.first.first); }; std::sort(hits.begin(), hits.end(), sort_on_docid); + _searchItr.initRange(hits.front().first.first, hits.back().first.first + 1); for (auto &hit: hits) { hit.first.second = doScore(hit.first.first); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp b/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp index ca09a3e79f1..6115d642b4a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp @@ -32,8 +32,9 @@ auto extract_names(const FeatureResolver &resolver, const StringStringMap &renam result.reserve(resolver.num_features()); for (size_t i = 0; i < resolver.num_features(); ++i) { vespalib::string name = resolver.name_of(i); - if (renames.contains(name)) { - name = renames[name]; + auto iter = renames.find(name); + if (iter != renames.end()) { + name = iter->second; } result.emplace_back(name); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index f1243665636..102da1d71e4 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -276,9 +276,6 @@ MatchThread::findMatches(MatchTools &tools) match_loop_helper(tools, hits); if (tools.has_second_phase_rank()) { trace->addEvent(4, "Start second phase rerank"); - tools.setup_second_phase(); - DocidRange docid_range(1, matchParams.numDocs); - tools.search().initRange(docid_range.begin, docid_range.end); auto sorted_hit_seq = matchToolsFactory.should_diversify() ? hits.getSortedHitSequence(matchParams.arraySize) : hits.getSortedHitSequence(matchParams.heapSize); @@ -286,11 +283,14 @@ MatchThread::findMatches(MatchTools &tools) WaitTimer get_second_phase_work_timer(wait_time_s); auto my_work = communicator.get_second_phase_work(sorted_hit_seq, thread_id); get_second_phase_work_timer.done(); - DocumentScorer scorer(tools.rank_program(), tools.search()); if (tools.getDoom().hard_doom()) { my_work.clear(); } - scorer.score(my_work); + if (!my_work.empty()) { + tools.setup_second_phase(); + DocumentScorer scorer(tools.rank_program(), tools.search()); + scorer.score(my_work); + } thread_stats.docsReRanked(my_work.size()); trace->addEvent(5, "Synchronize before rank scaling"); WaitTimer complete_second_phase_timer(wait_time_s); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 88d5281535a..826860b743e 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -196,8 +196,8 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl { // we want to measure full set-up and tear-down time as part of // collateral time GroupingContext groupingContext(_clock, request.getTimeOfDoom(), - &request.groupSpec[0], request.groupSpec.size()); - SessionId sessionId(&request.sessionId[0], request.sessionId.size()); + request.groupSpec.data(), request.groupSpec.size()); + SessionId sessionId(request.sessionId.data(), request.sessionId.size()); bool shouldCacheSearchSession = false; bool shouldCacheGroupingSession = false; if (!sessionId.empty()) { @@ -355,7 +355,7 @@ Matcher::create_docsum_matcher(const DocsumRequest &req, ISearchContext &search_ } } std::sort(docs.begin(), docs.end()); - SessionId sessionId(&req.sessionId[0], req.sessionId.size()); + SessionId sessionId(req.sessionId.data(), req.sessionId.size()); bool expectedSessionCached(false); if (!sessionId.empty()) { const Properties &cache_props = req.propertiesMap.cacheProperties(); diff --git a/searchcore/src/vespa/searchcore/proton/matching/partial_result.cpp b/searchcore/src/vespa/searchcore/proton/matching/partial_result.cpp index 432752d69d0..450779dd97d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/partial_result.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/partial_result.cpp @@ -20,9 +20,9 @@ void mergeHits(size_t maxHits, std::vector<search::RankedHit> my_hits; std::swap(hits, my_hits); hits.reserve(maxHits); - const search::RankedHit *a_pos = &my_hits[0]; + const search::RankedHit *a_pos = my_hits.data(); const search::RankedHit *a_end = a_pos + my_hits.size(); - const search::RankedHit *b_pos = &rhs_hits[0]; + const search::RankedHit *b_pos = rhs_hits.data(); const search::RankedHit *b_end = b_pos + rhs_hits.size(); while (a_pos < a_end && b_pos < b_end && hits.size() < maxHits) { if (before(*a_pos, *b_pos)) { @@ -64,12 +64,12 @@ size_t mergeHits(size_t maxHits, std::swap(sortData, my_sortData); hits.reserve(maxHits); sortData.reserve(maxHits); - const search::RankedHit *a_pos = &my_hits[0]; + const search::RankedHit *a_pos = my_hits.data(); const search::RankedHit *a_end = a_pos + my_hits.size(); - const search::RankedHit *b_pos = &rhs_hits[0]; + const search::RankedHit *b_pos = rhs_hits.data(); const search::RankedHit *b_end = b_pos + rhs_hits.size(); - const PartialResult::SortRef *a_sort_pos = &my_sortData[0]; - const PartialResult::SortRef *b_sort_pos = &rhs_sortData[0]; + const PartialResult::SortRef *a_sort_pos = my_sortData.data(); + const PartialResult::SortRef *b_sort_pos = rhs_sortData.data(); while (a_pos < a_end && b_pos < b_end && hits.size() < maxHits) { if (before(*a_sort_pos, a_pos->_docId, *b_sort_pos, b_pos->_docId)) diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index 4da09df1297..820e61bc7a7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -131,7 +131,7 @@ ConfigFile::ConfigFile(const vespalib::string &name, const vespalib::string &ful return; int64_t fileSize = file.getSize(); _content.resize(fileSize); - file.ReadBuf(&_content[0], fileSize); + file.ReadBuf(_content.data(), fileSize); _modTime = file.GetModificationTime(); } @@ -143,7 +143,7 @@ ConfigFile::serialize(nbostream &stream) const stream << static_cast<int64_t>(_modTime);; uint32_t sz = _content.size(); stream << sz; - stream.write(&_content[0], sz); + stream.write(_content.data(), sz); return stream; } @@ -159,7 +159,9 @@ ConfigFile::deserialize(nbostream &stream) stream >> sz; _content.resize(sz); assert(stream.size() >= sz); - memcpy(&_content[0], stream.peek(), sz); + if (sz > 0) { + memcpy(_content.data(), stream.peek(), sz); + } stream.adjustReadPos(sz); return stream; } @@ -173,7 +175,7 @@ ConfigFile::save(const vespalib::string &snapDir) const assert(openRes); (void) openRes; - file.WriteBuf(&_content[0], _content.size()); + file.WriteBuf(_content.data(), _content.size()); bool closeRes = file.Close(); assert(closeRes); (void) closeRes; @@ -387,7 +389,7 @@ FileConfigManager::removeInvalid() vespalib::string snapDirBaseName(makeSnapDirBaseName(serial)); vespalib::string snapDir(_baseDir + "/" + snapDirBaseName); try { - FastOS_FileInterface::EmptyAndRemoveDirectory(snapDir.c_str()); + std::filesystem::remove_all(std::filesystem::path(snapDir)); } catch (const std::exception &e) { LOG(warning, "Removing obsolete config directory '%s' failed due to %s", snapDir.c_str(), e.what()); } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index ff5924fa6bc..c4b30d50dbd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -149,8 +149,18 @@ const vespalib::string CUSTOM_COMPONENT_API_PATH = "/state/v1/custom/component"; VESPA_THREAD_STACK_TAG(proton_initialize_executor) VESPA_THREAD_STACK_TAG(proton_close_executor) +void ensureWritableDir(const vespalib::string &dirName) { + auto filename = dirName + "/tmp.filesystem.probe"; + vespalib::File probe(filename); + probe.unlink(); + probe.open(vespalib::File::CREATE); + probe.write("probe\n", 6, 0); + probe.close(); + probe.unlink(); } +} // namespace <unnamed> + Proton::ProtonFileHeaderContext::ProtonFileHeaderContext(const vespalib::string &creator) : _hostName(), _creator(creator), @@ -278,6 +288,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) { assert( _initStarted && ! _initComplete ); const ProtonConfig &protonConfig = configSnapshot->getProtonConfig(); + ensureWritableDir(protonConfig.basedir); const HwInfo & hwInfo = configSnapshot->getHwInfo(); _hw_info = hwInfo; diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp index 86db96a20ac..5f522bfaffc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/nice.h> using vespalib::CpuUsage; using vespalib::steady_time; @@ -24,7 +25,7 @@ SharedThreadingService::SharedThreadingService(const SharedThreadingServiceConfi CpuUsage::wrap(proton_warmup_executor, CpuUsage::Category::COMPACT), cfg.shared_task_limit())), _shared(std::make_shared<vespalib::BlockingThreadStackExecutor>(cfg.shared_threads(), 128_Ki, - cfg.shared_task_limit(), proton_shared_executor)), + cfg.shared_task_limit(), vespalib::be_nice(proton_shared_executor, cfg.feeding_niceness()))), _field_writer(), _invokeService(std::max(vespalib::adjustTimeoutByDetectedHz(1ms), cfg.field_writer_config().reactionTime())), @@ -33,7 +34,7 @@ SharedThreadingService::SharedThreadingService(const SharedThreadingServiceConfi _clock(_invokeService.nowRef()) { const auto& fw_cfg = cfg.field_writer_config(); - _field_writer = vespalib::SequencedTaskExecutor::create(CpuUsage::wrap(proton_field_writer_executor, CpuUsage::Category::WRITE), + _field_writer = vespalib::SequencedTaskExecutor::create(vespalib::be_nice(CpuUsage::wrap(proton_field_writer_executor, CpuUsage::Category::WRITE), cfg.feeding_niceness()), cfg.field_writer_threads(), fw_cfg.defaultTaskLimit(), fw_cfg.is_task_limit_hard(), diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp index 002ac508b4a..50ef5039e75 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp @@ -12,11 +12,13 @@ SharedThreadingServiceConfig::SharedThreadingServiceConfig(uint32_t shared_threa uint32_t shared_task_limit_in, uint32_t warmup_threads_in, uint32_t field_writer_threads_in, + double feeding_niceness_in, const ThreadingServiceConfig& field_writer_config_in) : _shared_threads(shared_threads_in), _shared_task_limit(shared_task_limit_in), _warmup_threads(warmup_threads_in), _field_writer_threads(field_writer_threads_in), + _feeding_niceness(feeding_niceness_in), _field_writer_config(field_writer_config_in) { } @@ -61,6 +63,7 @@ SharedThreadingServiceConfig::make(const proton::SharedThreadingServiceConfig::P return proton::SharedThreadingServiceConfig(shared_threads, shared_threads * 16, derive_warmup_threads(cpu_info), field_writer_threads, + cfg.feeding.niceness, ThreadingServiceConfig::make(cfg)); } diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.h b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.h index 5a2468ca1ab..add592f463d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.h @@ -20,6 +20,7 @@ private: uint32_t _shared_task_limit; uint32_t _warmup_threads; uint32_t _field_writer_threads; + double _feeding_niceness; ThreadingServiceConfig _field_writer_config; public: @@ -27,6 +28,7 @@ public: uint32_t shared_task_limit_in, uint32_t warmup_threads_in, uint32_t field_writer_threads_in, + double feed_niceness_in, const ThreadingServiceConfig& field_writer_config_in); static SharedThreadingServiceConfig make(const ProtonConfig& cfg, const HwInfo::Cpu& cpu_info); @@ -35,6 +37,7 @@ public: uint32_t shared_task_limit() const { return _shared_task_limit; } uint32_t warmup_threads() const { return _warmup_threads; } uint32_t field_writer_threads() const { return _field_writer_threads; } + double feeding_niceness() const { return _feeding_niceness; } const ThreadingServiceConfig& field_writer_config() const { return _field_writer_config; } }; diff --git a/searchcore/src/vespa/searchcorespi/index/fusionrunner.cpp b/searchcore/src/vespa/searchcorespi/index/fusionrunner.cpp index 1675b6091cf..1d63e883245 100644 --- a/searchcore/src/vespa/searchcorespi/index/fusionrunner.cpp +++ b/searchcore/src/vespa/searchcorespi/index/fusionrunner.cpp @@ -6,7 +6,6 @@ #include <vespa/searchlib/common/serialnumfileheadercontext.h> #include <vespa/searchlib/attribute/fixedsourceselector.h> #include <vespa/searchlib/queryeval/isourceselector.h> -#include <vespa/searchlib/util/dirtraverse.h> #include <vespa/log/log.h> LOG_SETUP(".searchcorespi.index.fusionrunner"); diff --git a/searchcore/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcore/src/vespa/searchcorespi/index/indexmaintainer.cpp index af273bc5e45..1f022b1215b 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -13,7 +13,6 @@ #include <vespa/searchcorespi/flush/lambdaflushtask.h> #include <vespa/searchlib/common/i_flush_token.h> #include <vespa/searchlib/index/schemautil.h> -#include <vespa/searchlib/util/dirtraverse.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/array.hpp> @@ -23,6 +22,7 @@ #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/time.h> #include <vespa/fastos/file.h> +#include <filesystem> #include <sstream> #include <vespa/log/log.h> @@ -1103,7 +1103,7 @@ IndexMaintainer::runFusion(const FusionSpec &fusion_spec, std::shared_ptr<search } else { LOG(error, "Fusion failed, fusion dir \"%s\".", fail_dir.c_str()); } - FastOS_FileInterface::EmptyAndRemoveDirectory(fail_dir.c_str()); + std::filesystem::remove_all(std::filesystem::path(fail_dir)); { LockGuard slock(_state_lock); LockGuard ilock(_index_update_lock); diff --git a/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp b/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp index cc2575f74d2..54d2cf6e1f5 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp @@ -7,7 +7,9 @@ #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/exceptions.h> +#include <filesystem> #include <sstream> +#include <system_error> #include <unistd.h> #include <vespa/log/log.h> @@ -80,7 +82,10 @@ IndexWriteUtilities::copySerialNumFile(const vespalib::string &sourceDir, vespalib::string source = IndexDiskLayout::getSerialNumFileName(sourceDir); vespalib::string dest = IndexDiskLayout::getSerialNumFileName(destDir); vespalib::string tmpDest = dest + ".tmp"; - if (!FastOS_FileInterface::CopyFile(source.c_str(), tmpDest.c_str())) { + std::error_code ec; + + std::filesystem::copy_file(std::filesystem::path(source), std::filesystem::path(tmpDest), ec); + if (ec) { LOG(error, "Unable to copy file '%s'", source.c_str()); return false; } diff --git a/searchlib/src/apps/tests/biglogtest.cpp b/searchlib/src/apps/tests/biglogtest.cpp index d5c59bf5b29..bd8991edc4b 100644 --- a/searchlib/src/apps/tests/biglogtest.cpp +++ b/searchlib/src/apps/tests/biglogtest.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/data/databuffer.h> +#include <filesystem> using namespace search; using search::index::DummyFileHeaderContext; @@ -148,9 +149,8 @@ Test::testDIO() { uint64_t serial = 0; - FastOS_File::EmptyDirectory(_dir.c_str()); - FastOS_File::RemoveDirectory(_dir.c_str()); - EXPECT_TRUE(FastOS_File::MakeDirectory(_dir.c_str())); + std::filesystem::remove_all(std::filesystem::path(_dir)); + std::filesystem::create_directory(std::filesystem::path(_dir)); Map lidToBlobMap; vespalib::DataBuffer buf; @@ -238,7 +238,6 @@ Test::testDIO() factory<DS> ds(_dir); checkBlobs(ds(), lidToBlobMap); } - FastOS_File::EmptyDirectory(_dir.c_str()); - FastOS_File::RemoveDirectory(_dir.c_str()); + std::filesystem::remove_all(std::filesystem::path(_dir)); TEST_FLUSH(); } diff --git a/searchlib/src/tests/alignment/alignment.cpp b/searchlib/src/tests/alignment/alignment.cpp index 06acf96e16c..3c6906f45bf 100644 --- a/searchlib/src/tests/alignment/alignment.cpp +++ b/searchlib/src/tests/alignment/alignment.cpp @@ -6,6 +6,9 @@ LOG_SETUP("alignment_test"); #include <sys/time.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/memory.h> + +using vespalib::Unaligned; struct Timer { rusage usage; @@ -28,7 +31,7 @@ TEST_SETUP(Test); double timeAccess(void *bufp, uint32_t len, double &sum) { - double *buf = (double *)bufp; + auto buf = Unaligned<double>::ptr(bufp); Timer timer; timer.start(); for(uint32_t i = 0; i < 512_Ki; ++i) { diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index bb516c3b451..a0906e2a488 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -791,7 +791,7 @@ AttributeTest::checkCount(const AttributePtr & ptr, uint32_t doc, uint32_t value if (!result) { return false; } - EXPECT_EQ(valueCount, ptr->get(doc, &buffer[0], buffer.size())) << (result = false, ""); + EXPECT_EQ(valueCount, ptr->get(doc, buffer.data(), buffer.size())) << (result = false, ""); if (!result) { return false; } @@ -807,7 +807,7 @@ AttributeTest::checkContent(const AttributePtr & ptr, uint32_t doc, uint32_t val std::vector<BufferType> buffer(valueCount); bool retval = true; EXPECT_TRUE((retval = retval && (static_cast<uint32_t>(ptr->getValueCount(doc)) == valueCount))); - EXPECT_TRUE((retval = retval && (ptr->get(doc, &buffer[0], buffer.size()) == valueCount))); + EXPECT_TRUE((retval = retval && (ptr->get(doc, buffer.data(), buffer.size()) == valueCount))); for (uint32_t i = 0; i < valueCount; ++i) { EXPECT_TRUE((retval = retval && (buffer[i] == values[i % range]))); } @@ -868,7 +868,7 @@ AttributeTest::testSingle(const AttributePtr & ptr, const std::vector<BufferType ptr->clearDoc(doc); } ptr->commit(); - EXPECT_EQ(1u, ptr->get(doc, &buffer[0], buffer.size())); + EXPECT_EQ(1u, ptr->get(doc, buffer.data(), buffer.size())); if (doc % 2 == 0) { if (smallUInt) { expectZero(buffer[0]); @@ -1156,7 +1156,7 @@ AttributeTest::testWeightedSet(const AttributePtr & ptr, const std::vector<Buffe EXPECT_TRUE(v.append(doc, values[j].getValue(), values[j].getWeight())); } commit(ptr); - ASSERT_TRUE(ptr->get(doc, &buffer[0], buffer.size()) == valueCount); + ASSERT_TRUE(ptr->get(doc, buffer.data(), buffer.size()) == valueCount); std::sort(buffer.begin(), buffer.begin() + valueCount, order_by_weight()); for (uint32_t j = 0; j < valueCount; ++j) { EXPECT_TRUE(buffer[j].getValue() == ordered_values[j].getValue()); @@ -1173,20 +1173,20 @@ AttributeTest::testWeightedSet(const AttributePtr & ptr, const std::vector<Buffe // append non-existent value EXPECT_TRUE(v.append(doc, values[doc].getValue(), values[doc].getWeight())); commit(ptr); - ASSERT_TRUE(ptr->get(doc, &buffer[0], buffer.size()) == valueCount + 1); + ASSERT_TRUE(ptr->get(doc, buffer.data(), buffer.size()) == valueCount + 1); EXPECT_TRUE(contains(buffer, valueCount + 1, values[doc])); // append existent value EXPECT_TRUE(v.append(doc, values[doc].getValue(), values[doc].getWeight() + 10)); commit(ptr); - ASSERT_TRUE(ptr->get(doc, &buffer[0], buffer.size()) == valueCount + 1); + ASSERT_TRUE(ptr->get(doc, buffer.data(), buffer.size()) == valueCount + 1); EXPECT_TRUE(contains(buffer, valueCount + 1, BufferType(values[doc].getValue(), values[doc].getWeight() + 10))); // append non-existent value two times EXPECT_TRUE(v.append(doc, values[doc + 1].getValue(), values[doc + 1].getWeight())); EXPECT_TRUE(v.append(doc, values[doc + 1].getValue(), values[doc + 1].getWeight() + 10)); commit(ptr); - ASSERT_TRUE(ptr->get(doc, &buffer[0], buffer.size()) == valueCount + 2); + ASSERT_TRUE(ptr->get(doc, buffer.data(), buffer.size()) == valueCount + 2); EXPECT_TRUE(contains(buffer, valueCount + 2, BufferType(values[doc + 1].getValue(), values[doc + 1].getWeight() + 10))); } EXPECT_EQ(ptr->getStatus().getUpdateCount(), numDocs + (numDocs*(numDocs-1))/2 + numDocs*4); @@ -1203,11 +1203,11 @@ AttributeTest::testWeightedSet(const AttributePtr & ptr, const std::vector<Buffe EXPECT_TRUE(static_cast<uint32_t>(v.getValueCount(doc)) == valueCount + 2); // remove existent value - ASSERT_TRUE(ptr->get(doc, &buffer[0], buffer.size()) == valueCount + 2); + ASSERT_TRUE(ptr->get(doc, buffer.data(), buffer.size()) == valueCount + 2); EXPECT_TRUE(contains_value(buffer, valueCount + 2, values[doc + 1].getValue())); EXPECT_TRUE(v.remove(doc, values[doc + 1].getValue(), 0)); commit(ptr); - ASSERT_TRUE(ptr->get(doc, &buffer[0], buffer.size()) == valueCount + 1); + ASSERT_TRUE(ptr->get(doc, buffer.data(), buffer.size()) == valueCount + 1); EXPECT_FALSE(contains_value(buffer, valueCount + 1, values[doc + 1].getValue())); } EXPECT_EQ(ptr->getStatus().getUpdateCount(), numDocs + (numDocs*(numDocs-1))/2 + numDocs*4 + numDocs * 2); diff --git a/searchlib/src/tests/attribute/document_weight_or_filter_search/document_weight_or_filter_search_test.cpp b/searchlib/src/tests/attribute/document_weight_or_filter_search/document_weight_or_filter_search_test.cpp index 5c1c49d8eb5..e27065f1c25 100644 --- a/searchlib/src/tests/attribute/document_weight_or_filter_search/document_weight_or_filter_search_test.cpp +++ b/searchlib/src/tests/attribute/document_weight_or_filter_search/document_weight_or_filter_search_test.cpp @@ -47,7 +47,7 @@ public: for (auto& key : keys) { adds.emplace_back(KeyData(key, 1)); } - _postings.apply(_trees[idx], &*adds.begin(), &*adds.end(), &*removes.begin(), &*removes.end()); + _postings.apply(_trees[idx], adds.data(), adds.data() + adds.size(), removes.data(), removes.data() + removes.size()); } void clear_tree(size_t idx) { diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index 92c3da40fe9..90127e9ae7b 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -21,6 +21,7 @@ #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/compress.h> +#include <vespa/vespalib/util/memory.h> #include <vespa/vespalib/stllike/asciistream.h> #include <limits> #include <cmath> @@ -184,8 +185,8 @@ MemAttr::bufEqual(const Buffer &lhs, const Buffer &rhs) const return true; if (!EXPECT_TRUE(lhs->getDataLen() == rhs->getDataLen())) return false; - if (!EXPECT_TRUE(memcmp(lhs->getData(), rhs->getData(), - lhs->getDataLen()) == 0)) + if (!EXPECT_TRUE(vespalib::memcmp_safe(lhs->getData(), rhs->getData(), + lhs->getDataLen()) == 0)) return false; return true; } @@ -480,7 +481,7 @@ EnumeratedSaveTest::getSearch(const V &vec, const T &term, bool prefix) buildTermQuery(query, vec.getName(), ss.str(), prefix); return (static_cast<const AttributeVector &>(vec)). - getSearch(vespalib::stringref(&query[0], query.size()), + getSearch(vespalib::stringref(query.data(), query.size()), SearchContextParams()); } diff --git a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp index 10cc14012dd..573284ffa35 100644 --- a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp +++ b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp @@ -83,8 +83,8 @@ protected: additions.emplace_back(i, 0); } _store.apply(root, - &additions[0], &additions[0] + additions.size(), - &removals[0], &removals[0] + removals.size()); + additions.data(), additions.data() + additions.size(), + removals.data(), removals.data() + removals.size()); return root; } static std::vector<int> make_exp_sequence(int start_key, int end_key) diff --git a/searchlib/src/tests/attribute/postinglist/postinglist.cpp b/searchlib/src/tests/attribute/postinglist/postinglist.cpp index 446aeaf22a7..54efb3261c8 100644 --- a/searchlib/src/tests/attribute/postinglist/postinglist.cpp +++ b/searchlib/src/tests/attribute/postinglist/postinglist.cpp @@ -379,8 +379,8 @@ insertRandomValues(Tree &tree, std::vector<AttributePosting> additions; std::vector<uint32_t> removals; additions.push_back(newPosting); - postings.apply(newIdx, &additions[0], &additions[0] + additions.size(), - &removals[0], &removals[0] + removals.size()); + postings.apply(newIdx, additions.data(), additions.data() + additions.size(), + removals.data(), removals.data() + removals.size()); std::atomic_thread_fence(std::memory_order_release); itr.writeData(newIdx); @@ -461,8 +461,8 @@ removeRandomValues(Tree &tree, std::vector<AttributePosting> additions; std::vector<uint32_t> removals; removals.push_back(i->_docId); - postings.apply(newIdx, &additions[0], &additions[0]+additions.size(), - &removals[0], &removals[0] + removals.size()); + postings.apply(newIdx, additions.data(), additions.data() + additions.size(), + removals.data(), removals.data() + removals.size()); if (newIdx != oldIdx) { std::atomic_thread_fence(std::memory_order_release); itr.writeData(newIdx); diff --git a/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp b/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp index c6091604a97..4ca2802d22d 100644 --- a/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp +++ b/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp @@ -6,7 +6,7 @@ #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/fastos/file.h> +#include <filesystem> #include <vespa/log/log.h> LOG_SETUP("sourceselector_test"); @@ -161,8 +161,8 @@ Test::requireThatSelectorCanSaveAndLoad(bool compactLidSpace) selector.compactLidSpace(maxDocId - 4); } - FastOS_FileInterface::EmptyAndRemoveDirectory(index_dir.c_str()); - FastOS_FileInterface::MakeDirIfNotPresentOrExit(index_dir.c_str()); + std::filesystem::remove_all(std::filesystem::path(index_dir)); + std::filesystem::create_directory(std::filesystem::path(index_dir)); SourceSelector::SaveInfo::UP save_info = selector.extractSaveInfo(base_file_name); @@ -177,7 +177,7 @@ Test::requireThatSelectorCanSaveAndLoad(bool compactLidSpace) EXPECT_EQUAL(maxDocId + 2, selector2->getDocIdLimit()); } - FastOS_FileInterface::EmptyAndRemoveDirectory(index_dir.c_str()); + std::filesystem::remove_all(std::filesystem::path(index_dir)); } void diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index aa2c475e7b6..96039bee15b 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -48,7 +48,7 @@ checkCount(Attribute & vec, uint32_t doc, uint32_t valueCount, { std::vector<vespalib::string> buffer(valueCount); EXPECT_TRUE(static_cast<uint32_t>(vec.getValueCount(doc)) == valueCount); - EXPECT_TRUE(vec.get(doc, &buffer[0], buffer.size()) == valueCount); + EXPECT_TRUE(vec.get(doc, buffer.data(), buffer.size()) == valueCount); EXPECT_TRUE(std::count(buffer.begin(), buffer.end(), value) == numValues); } @@ -125,10 +125,10 @@ testMultiValue(Attribute & attr, uint32_t numDocs) // test get all std::vector<vespalib::string> values(valueCount); - ASSERT_TRUE(attr.get(doc, &values[0], valueCount) == valueCount); + ASSERT_TRUE(attr.get(doc, values.data(), valueCount) == valueCount); std::vector<uint32_t> enums(valueCount); - ASSERT_TRUE((static_cast<search::attribute::IAttributeVector &>(attr)).get(doc, &enums[0], valueCount) == valueCount); + ASSERT_TRUE((static_cast<search::attribute::IAttributeVector &>(attr)).get(doc, enums.data(), valueCount) == valueCount); auto combined = zipped_and_sorted_by_first(values, enums); for (uint32_t j = 0; j < valueCount; ++j) { @@ -167,10 +167,10 @@ testMultiValue(Attribute & attr, uint32_t numDocs) // test get all std::vector<vespalib::string> values(valueCount); - EXPECT_TRUE(attr.get(doc, &values[0], valueCount) == valueCount); + EXPECT_TRUE(attr.get(doc, values.data(), valueCount) == valueCount); std::vector<uint32_t> enums(valueCount); - EXPECT_TRUE((static_cast<search::attribute::IAttributeVector &>(attr)).get(doc, &enums[0], valueCount) == valueCount); + EXPECT_TRUE((static_cast<search::attribute::IAttributeVector &>(attr)).get(doc, enums.data(), valueCount) == valueCount); auto combined = zipped_and_sorted_by_first(values, enums); for (uint32_t j = 0; j < valueCount; ++j) { diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index bf0b74b0003..72b2f1e320a 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -157,6 +157,12 @@ public: void expect_empty_add() const { EXPECT_TRUE(_adds.empty()); } + void expect_empty_prepare_add() const { + EXPECT_TRUE(_prepare_adds.empty()); + } + void expect_empty_complete_add() const { + EXPECT_TRUE(_complete_adds.empty()); + } void expect_entry(uint32_t exp_docid, const DoubleVector& exp_vector, const EntryVector& entries) const { EXPECT_EQUAL(1u, entries.size()); EXPECT_EQUAL(exp_docid, entries.back().first); @@ -881,6 +887,30 @@ TEST_F("nearest neighbor index can be updated in two phases", DenseTensorAttribu } } +TEST_F("nearest neighbor index is NOT updated when tensor value is unchanged", DenseTensorAttributeMockIndex) +{ + auto& index = f.mock_index(); + { + auto vec_a = vec_2d(3, 5); + auto prepare_result = f.prepare_set_tensor(1, vec_a); + index.expect_prepare_add(1, {3, 5}); + f.complete_set_tensor(1, vec_a, std::move(prepare_result)); + f.assertGetTensor(vec_a, 1); + index.expect_complete_add(1, {3, 5}); + } + index.clear(); + { + // Replaces previous value with the same value + auto vec_b = vec_2d(3, 5); + auto prepare_result = f.prepare_set_tensor(1, vec_b); + EXPECT_TRUE(prepare_result.get() == nullptr); + index.expect_empty_prepare_add(); + f.complete_set_tensor(1, vec_b, std::move(prepare_result)); + f.assertGetTensor(vec_b, 1); + index.expect_empty_complete_add(); + } +} + TEST_F("clearDoc() updates nearest neighbor index", DenseTensorAttributeMockIndex) { auto& index = f.mock_index(); diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index fffa1778c85..418182f7bbf 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -18,6 +18,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/memory.h> #include <iomanip> using document::BucketId; @@ -275,7 +276,7 @@ void fetchAndTest(IDataStore & datastore, uint32_t lid, const void *a, size_t sz vespalib::DataBuffer buf; EXPECT_EQUAL(static_cast<ssize_t>(sz), datastore.read(lid, buf)); EXPECT_EQUAL(buf.getDataLen(), sz); - EXPECT_TRUE(memcmp(a, buf.getData(), sz) == 0); + EXPECT_TRUE(vespalib::memcmp_safe(a, buf.getData(), sz) == 0); } TEST("testTruncatedIdxFile"){ @@ -666,13 +667,13 @@ TEST("test that the integrated visit cache works.") { } TEST("testWriteRead") { - FastOS_File::RemoveDirectory("empty"); + std::filesystem::remove_all(std::filesystem::path("empty")); const char * bufA = "aaaaaaaaaaaaaaaaaaaaa"; const char * bufB = "bbbbbbbbbbbbbbbb"; const vespalib::ConstBufferRef a[2] = { vespalib::ConstBufferRef(bufA, strlen(bufA)), vespalib::ConstBufferRef(bufB, strlen(bufB))}; LogDataStore::Config config; { - EXPECT_TRUE(FastOS_File::MakeDirectory("empty")); + std::filesystem::create_directory(std::filesystem::path("empty")); DummyFileHeaderContext fileHeaderContext; vespalib::ThreadStackExecutor executor(1, 128_Ki); MyTlSyncer tlSyncer; @@ -736,7 +737,7 @@ TEST("testWriteRead") { EXPECT_EQUAL(0ul, datastore.getDiskBloat()); EXPECT_EQUAL(0ul, datastore.getMaxSpreadAsBloat()); } - FastOS_File::EmptyAndRemoveDirectory("empty"); + std::filesystem::remove_all(std::filesystem::path("empty")); } TEST("requireThatSyncTokenIsUpdatedAfterFlush") { diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index d478adafa57..87de62dbfad 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -2,6 +2,7 @@ #include "mysearch.h" #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchlib/queryeval/isourceselector.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> @@ -23,6 +24,14 @@ using namespace search::fef; using namespace search::query; using search::BitVector; +struct InvalidSelector : ISourceSelector { + InvalidSelector() : ISourceSelector(Source()) {} + void setSource(uint32_t, Source) override { abort(); } + uint32_t getDocIdLimit() const override { abort(); } + void compactLidSpace(uint32_t) override { abort(); } + std::unique_ptr<sourceselector::Iterator> createIterator() const override { abort(); } +}; + struct WeightOrder { bool operator()(const wand::Term &t1, const wand::Term &t2) const { return (t1.weight < t2.weight); @@ -412,7 +421,7 @@ TEST("test Rank Blueprint") { } TEST("test SourceBlender Blueprint") { - ISourceSelector *selector = nullptr; // not needed here + auto selector = std::make_unique<InvalidSelector>(); // not needed here SourceBlenderBlueprint b(*selector); { // combine std::vector<Blueprint::HitEstimate> est; @@ -485,8 +494,8 @@ TEST("test SourceBlender Blueprint") { } TEST("test SourceBlender below AND optimization") { - ISourceSelector *selector_1 = 0; // the one - ISourceSelector *selector_2 = reinterpret_cast<ISourceSelector*>(100); // not the one + auto selector_1 = std::make_unique<InvalidSelector>(); // the one + auto selector_2 = std::make_unique<InvalidSelector>(); // not the one //------------------------------------------------------------------------- AndBlueprint *top = new AndBlueprint(); Blueprint::UP top_bp(top); @@ -567,8 +576,8 @@ TEST("test SourceBlender below AND optimization") { } TEST("test SourceBlender below OR optimization") { - ISourceSelector *selector_1 = 0; // the one - ISourceSelector *selector_2 = reinterpret_cast<ISourceSelector*>(100); // not the one + auto selector_1 = std::make_unique<InvalidSelector>(); // the one + auto selector_2 = std::make_unique<InvalidSelector>(); // not the one //------------------------------------------------------------------------- OrBlueprint *top = new OrBlueprint(); Blueprint::UP top_up(top); @@ -649,8 +658,8 @@ TEST("test SourceBlender below OR optimization") { } TEST("test SourceBlender below AND_NOT optimization") { - ISourceSelector *selector_1 = 0; // the one - ISourceSelector *selector_2 = reinterpret_cast<ISourceSelector*>(100); // not the one + auto selector_1 = std::make_unique<InvalidSelector>(); // the one + auto selector_2 = std::make_unique<InvalidSelector>(); // not the one //------------------------------------------------------------------------- AndNotBlueprint *top = new AndNotBlueprint(); Blueprint::UP top_up(top); @@ -741,8 +750,8 @@ TEST("test SourceBlender below AND_NOT optimization") { } TEST("test SourceBlender below RANK optimization") { - ISourceSelector *selector_1 = 0; // the one - ISourceSelector *selector_2 = reinterpret_cast<ISourceSelector*>(100); // not the one + auto selector_1 = std::make_unique<InvalidSelector>(); // the one + auto selector_2 = std::make_unique<InvalidSelector>(); // not the one //------------------------------------------------------------------------- RankBlueprint *top = new RankBlueprint(); Blueprint::UP top_up(top); @@ -876,7 +885,7 @@ TEST("test empty root node optimization and safeness") { } TEST("and with one empty child is optimized away") { - ISourceSelector *selector = 0; + auto selector = std::make_unique<InvalidSelector>(); Blueprint::UP top(ap((new SourceBlenderBlueprint(*selector))-> addChild(ap(MyLeafSpec(10).create())). addChild(ap((new AndBlueprint())-> @@ -891,7 +900,7 @@ TEST("and with one empty child is optimized away") { } TEST("test single child optimization") { - ISourceSelector *selector = 0; + auto selector = std::make_unique<InvalidSelector>(); //------------------------------------------------------------------------- Blueprint::UP top_up( ap((new AndNotBlueprint())-> diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/CMakeLists.txt b/searchlib/src/tests/queryeval/nearest_neighbor/CMakeLists.txt index 3ebc8eb5251..e543a847498 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/nearest_neighbor/CMakeLists.txt @@ -5,5 +5,6 @@ vespa_add_executable(searchlib_nearest_neighbor_test_app TEST nearest_neighbor_test.cpp DEPENDS searchlib + GTest::GTest ) vespa_add_test(NAME searchlib_nearest_neighbor_test_app COMMAND searchlib_nearest_neighbor_test_app) diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index 029b74ff914..1e341eab707 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -1,37 +1,38 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/stringfmt.h> - #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/feature.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/queryeval/nearest_neighbor_iterator.h> +#include <vespa/searchlib/queryeval/nns_index_iterator.h> #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> +#include <vespa/searchlib/tensor/distance_calculator.h> #include <vespa/searchlib/tensor/distance_function_factory.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/searchlib/queryeval/nns_index_iterator.h> -#include <vespa/searchcommon/attribute/config.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP("nearest_neighbor_test"); #define EPS 1.0e-6 -using search::feature_t; -using search::tensor::DenseTensorAttribute; using search::AttributeVector; using search::BitVector; -using vespalib::eval::Value; -using vespalib::eval::ValueType; +using search::attribute::DistanceMetric; +using search::feature_t; +using search::tensor::DenseTensorAttribute; +using search::tensor::DistanceCalculator; +using search::tensor::DistanceFunction; using vespalib::eval::CellType; -using vespalib::eval::TensorSpec; using vespalib::eval::SimpleValue; -using search::tensor::DistanceFunction; -using search::attribute::DistanceMetric; +using vespalib::eval::TensorSpec; +using vespalib::eval::Value; +using vespalib::eval::ValueType; using namespace search::fef; using namespace search::queryeval; @@ -96,7 +97,7 @@ struct Fixture uint32_t sz = _attr->getNumDocs(); _global_filter = BitVector::create(sz); for (uint32_t id : docids) { - EXPECT_LESS(id, sz); + EXPECT_LT(id, sz); _global_filter->setBit(id); } } @@ -112,11 +113,11 @@ struct Fixture setTensor(docId, *t); } - const DistanceFunction *dist_fun() const { + const DistanceFunction &dist_fun() const { if (_cfg.tensorType().cell_type() == CellType::FLOAT) { - return euclid_f.get(); + return *euclid_f; } else { - return euclid_d.get(); + return *euclid_d; } } }; @@ -126,10 +127,11 @@ SimpleResult find_matches(Fixture &env, const Value &qtv, double threshold = std auto md = MatchData::makeTestInstance(2, 2); auto &tfmd = *(md->resolveTermField(0)); auto &attr = *(env._tensorAttr); + DistanceCalculator dist_calc(attr, qtv, env.dist_fun()); NearestNeighborDistanceHeap dh(2); - dh.set_distance_threshold(env.dist_fun()->convert_threshold(threshold)); + dh.set_distance_threshold(env.dist_fun().convert_threshold(threshold)); const BitVector *filter = env._global_filter.get(); - auto search = NearestNeighborIterator::create(strict, tfmd, qtv, attr, dh, filter, env.dist_fun()); + auto search = NearestNeighborIterator::create(strict, tfmd, dist_calc, dh, filter); if (strict) { return SimpleResult().searchStrict(*search, attr.getNumDocs()); } else { @@ -152,33 +154,33 @@ verify_iterator_returns_expected_results(const vespalib::string& attribute_tenso auto nullTensor = createTensor(query_tensor_type_spec, 0.0, 0.0); SimpleResult result = find_matches<true>(fixture, *nullTensor); SimpleResult nullExpect({1,2,4,6}); - EXPECT_EQUAL(result, nullExpect); + EXPECT_EQ(result, nullExpect); result = find_matches<false>(fixture, *nullTensor); - EXPECT_EQUAL(result, nullExpect); + EXPECT_EQ(result, nullExpect); auto farTensor = createTensor(query_tensor_type_spec, 9.0, 9.0); SimpleResult farExpect({1,2,3,5}); result = find_matches<true>(fixture, *farTensor); - EXPECT_EQUAL(result, farExpect); + EXPECT_EQ(result, farExpect); result = find_matches<false>(fixture, *farTensor); - EXPECT_EQUAL(result, farExpect); + EXPECT_EQ(result, farExpect); SimpleResult null_thr5_exp({1,4,6}); result = find_matches<true>(fixture, *nullTensor, 5.0); - EXPECT_EQUAL(result, null_thr5_exp); + EXPECT_EQ(result, null_thr5_exp); result = find_matches<false>(fixture, *nullTensor, 5.0); - EXPECT_EQUAL(result, null_thr5_exp); + EXPECT_EQ(result, null_thr5_exp); SimpleResult far_thr4_exp({2,5}); result = find_matches<true>(fixture, *farTensor, 4.0); - EXPECT_EQUAL(result, far_thr4_exp); + EXPECT_EQ(result, far_thr4_exp); result = find_matches<false>(fixture, *farTensor, 4.0); - EXPECT_EQUAL(result, far_thr4_exp); + EXPECT_EQ(result, far_thr4_exp); } -TEST("require that NearestNeighborIterator returns expected results") { - TEST_DO(verify_iterator_returns_expected_results(denseSpecDouble, denseSpecDouble)); - TEST_DO(verify_iterator_returns_expected_results(denseSpecFloat, denseSpecFloat)); +TEST(NnsIndexIteratorTest, require_that_iterator_returns_expected_results) { + verify_iterator_returns_expected_results(denseSpecDouble, denseSpecDouble); + verify_iterator_returns_expected_results(denseSpecFloat, denseSpecFloat); } void @@ -197,20 +199,20 @@ verify_iterator_returns_filtered_results(const vespalib::string& attribute_tenso auto nullTensor = createTensor(query_tensor_type_spec, 0.0, 0.0); SimpleResult result = find_matches<true>(fixture, *nullTensor); SimpleResult nullExpect({1,3,4}); - EXPECT_EQUAL(result, nullExpect); + EXPECT_EQ(result, nullExpect); result = find_matches<false>(fixture, *nullTensor); - EXPECT_EQUAL(result, nullExpect); + EXPECT_EQ(result, nullExpect); auto farTensor = createTensor(query_tensor_type_spec, 9.0, 9.0); SimpleResult farExpect({1,3,4}); result = find_matches<true>(fixture, *farTensor); - EXPECT_EQUAL(result, farExpect); + EXPECT_EQ(result, farExpect); result = find_matches<false>(fixture, *farTensor); - EXPECT_EQUAL(result, farExpect); + EXPECT_EQ(result, farExpect); } -TEST("require that NearestNeighborIterator returns filtered results") { - TEST_DO(verify_iterator_returns_filtered_results(denseSpecDouble, denseSpecDouble)); - TEST_DO(verify_iterator_returns_filtered_results(denseSpecFloat, denseSpecFloat)); +TEST(NnsIndexIteratorTest, require_that_iterator_returns_filtered_results) { + verify_iterator_returns_filtered_results(denseSpecDouble, denseSpecDouble); + verify_iterator_returns_filtered_results(denseSpecFloat, denseSpecFloat); } template <bool strict> @@ -218,8 +220,9 @@ std::vector<feature_t> get_rawscores(Fixture &env, const Value &qtv) { auto md = MatchData::makeTestInstance(2, 2); auto &tfmd = *(md->resolveTermField(0)); auto &attr = *(env._tensorAttr); + DistanceCalculator dist_calc(attr, qtv, env.dist_fun()); NearestNeighborDistanceHeap dh(2); - auto search = NearestNeighborIterator::create(strict, tfmd, qtv, attr, dh, nullptr, env.dist_fun()); + auto search = NearestNeighborIterator::create(strict, tfmd, dist_calc, dh, nullptr); uint32_t limit = attr.getNumDocs(); uint32_t docid = 1; search->initRange(docid, limit); @@ -249,63 +252,63 @@ verify_iterator_sets_expected_rawscore(const vespalib::string& attribute_tensor_ auto nullTensor = createTensor(query_tensor_type_spec, 0.0, 0.0); std::vector<feature_t> got = get_rawscores<true>(fixture, *nullTensor); std::vector<feature_t> expected{5.0, 13.0, 10.0, 10.0, 5.0}; - EXPECT_EQUAL(got.size(), expected.size()); + EXPECT_EQ(got.size(), expected.size()); for (size_t i = 0; i < expected.size(); ++i) { - EXPECT_APPROX(1.0/(1.0+expected[i]), got[i], EPS); + EXPECT_NEAR(1.0/(1.0+expected[i]), got[i], EPS); } got = get_rawscores<false>(fixture, *nullTensor); - EXPECT_EQUAL(got.size(), expected.size()); + EXPECT_EQ(got.size(), expected.size()); for (size_t i = 0; i < expected.size(); ++i) { - EXPECT_APPROX(1.0/(1.0+expected[i]), got[i], EPS); + EXPECT_NEAR(1.0/(1.0+expected[i]), got[i], EPS); } } -TEST("require that NearestNeighborIterator sets expected rawscore") { - TEST_DO(verify_iterator_sets_expected_rawscore(denseSpecDouble, denseSpecDouble)); - TEST_DO(verify_iterator_sets_expected_rawscore(denseSpecFloat, denseSpecFloat)); +TEST(NnsIndexIteratorTest, require_that_iterator_sets_expected_rawscore) { + verify_iterator_sets_expected_rawscore(denseSpecDouble, denseSpecDouble); + verify_iterator_sets_expected_rawscore(denseSpecFloat, denseSpecFloat); } -TEST("require that NnsIndexIterator works as expected") { +TEST(NnsIndexIteratorTest, require_that_iterator_works_as_expected) { std::vector<NnsIndexIterator::Hit> hits{{2,4.0}, {3,9.0}, {5,1.0}, {8,16.0}, {9,36.0}}; auto md = MatchData::makeTestInstance(2, 2); auto &tfmd = *(md->resolveTermField(0)); - auto search = NnsIndexIterator::create(tfmd, hits, euclid_d.get()); + auto search = NnsIndexIterator::create(tfmd, hits, *euclid_d); uint32_t docid = 1; search->initFullRange(); bool match = search->seek(docid); EXPECT_FALSE(match); EXPECT_FALSE(search->isAtEnd()); - EXPECT_EQUAL(2u, search->getDocId()); + EXPECT_EQ(2u, search->getDocId()); docid = 2; match = search->seek(docid); EXPECT_TRUE(match); EXPECT_FALSE(search->isAtEnd()); - EXPECT_EQUAL(docid, search->getDocId()); + EXPECT_EQ(docid, search->getDocId()); search->unpack(docid); - EXPECT_APPROX(1.0/(1.0+2.0), tfmd.getRawScore(), EPS); + EXPECT_NEAR(1.0/(1.0+2.0), tfmd.getRawScore(), EPS); docid = 3; match = search->seek(docid); EXPECT_TRUE(match); EXPECT_FALSE(search->isAtEnd()); - EXPECT_EQUAL(docid, search->getDocId()); + EXPECT_EQ(docid, search->getDocId()); search->unpack(docid); - EXPECT_APPROX(1.0/(1.0+3.0), tfmd.getRawScore(), EPS); + EXPECT_NEAR(1.0/(1.0+3.0), tfmd.getRawScore(), EPS); docid = 4; match = search->seek(docid); EXPECT_FALSE(match); EXPECT_FALSE(search->isAtEnd()); - EXPECT_EQUAL(5u, search->getDocId()); + EXPECT_EQ(5u, search->getDocId()); docid = 6; match = search->seek(docid); EXPECT_FALSE(match); EXPECT_FALSE(search->isAtEnd()); - EXPECT_EQUAL(8u, search->getDocId()); + EXPECT_EQ(8u, search->getDocId()); docid = 8; search->unpack(docid); - EXPECT_APPROX(1.0/(1.0+4.0), tfmd.getRawScore(), EPS); + EXPECT_NEAR(1.0/(1.0+4.0), tfmd.getRawScore(), EPS); docid = 9; match = search->seek(docid); EXPECT_TRUE(match); @@ -320,10 +323,10 @@ TEST("require that NnsIndexIterator works as expected") { match = search->seek(docid); EXPECT_FALSE(match); EXPECT_FALSE(search->isAtEnd()); - EXPECT_EQUAL(5u, search->getDocId()); + EXPECT_EQ(5u, search->getDocId()); docid = 5; search->unpack(docid); - EXPECT_APPROX(1.0/(1.0+1.0), tfmd.getRawScore(), EPS); + EXPECT_NEAR(1.0/(1.0+1.0), tfmd.getRawScore(), EPS); EXPECT_FALSE(search->isAtEnd()); docid = 6; match = search->seek(docid); @@ -331,4 +334,4 @@ TEST("require that NnsIndexIterator works as expected") { EXPECT_TRUE(search->isAtEnd()); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_operation.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_operation.cpp index 57980237f21..55577b3916c 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_operation.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_operation.cpp @@ -111,7 +111,7 @@ public: void operator()(IAttributeVector &attributeVector) override { OP op(attributeVector, _operand); if (op.valid()) { - const RankedHit *hits = &_result.second[0]; + const RankedHit *hits = _result.second.data(); size_t numHits = _result.second.size(); std::for_each(hits, hits+numHits, [&op](RankedHit hit) { op(hit.getDocId()); }); if (_result.first) { diff --git a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.cpp b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.cpp index b514275f75d..80e9b28139a 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.cpp +++ b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.cpp @@ -13,7 +13,7 @@ sortLoadedByEnum(LoadedEnumAttributeVector &loaded) LoadedEnumAttribute::EnumCompare, 56>:: radix_sort(LoadedEnumAttribute::EnumRadix(), LoadedEnumAttribute::EnumCompare(), - &loaded[0], loaded.size(), 16); + loaded.data(), loaded.size(), 16); } } diff --git a/searchlib/src/vespa/searchlib/attribute/loadednumericvalue.cpp b/searchlib/src/vespa/searchlib/attribute/loadednumericvalue.cpp index 0ffd6e2c845..4d3912ae24d 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadednumericvalue.cpp +++ b/searchlib/src/vespa/searchlib/attribute/loadednumericvalue.cpp @@ -14,7 +14,7 @@ sortLoadedByValue(SequentialReadModifyWriteVector<LoadedNumericValue<T>> & loade typename LoadedNumericValue<T>::ValueCompare, 56>:: radix_sort(typename LoadedNumericValue<T>::ValueRadix(), typename LoadedNumericValue<T>::ValueCompare(), - &loaded[0], + loaded.data(), loaded.size(), 16); } @@ -29,7 +29,7 @@ sortLoadedByDocId(SequentialReadModifyWriteVector<LoadedNumericValue<T>> & loade typename LoadedNumericValue<T>::DocOrderCompare, 56>:: radix_sort(typename LoadedNumericValue<T>::DocRadix(), typename LoadedNumericValue<T>::DocOrderCompare(), - &loaded[0], + loaded.data(), loaded.size(), 16); } diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h index cc128b0eef1..0a29b4af48d 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h @@ -49,7 +49,7 @@ protected: using WType = MultiValueType; uint32_t get(DocId doc, const WType * & values) const { MultiValueArrayRef array(this->_mvMapping.get(doc)); - values = &array[0]; + values = array.data(); return array.size(); } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp index 0e0dceaf254..79276ce6f55 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp @@ -69,11 +69,11 @@ PostingListAttributeBase<P>::handle_load_posting_lists_and_update_enum_store(enu postings.removeDups(); newIndex = EntryRef(); _postingList.apply(newIndex, - &postings._additions[0], - &postings._additions[0] + + postings._additions.data(), + postings._additions.data() + postings._additions.size(), - &postings._removals[0], - &postings._removals[0] + + postings._removals.data(), + postings._removals.data() + postings._removals.size()); posting_indexes[posting_enum] = newIndex; postings.clear(); @@ -91,10 +91,10 @@ PostingListAttributeBase<P>::handle_load_posting_lists_and_update_enum_store(enu postings.removeDups(); newIndex = EntryRef(); _postingList.apply(newIndex, - &postings._additions[0], - &postings._additions[0] + postings._additions.size(), - &postings._removals[0], - &postings._removals[0] + postings._removals.size()); + postings._additions.data(), + postings._additions.data() + postings._additions.size(), + postings._removals.data(), + postings._removals.data() + postings._removals.size()); posting_indexes[posting_enum] = newIndex; loader.build_dictionary(); loader.free_unused_values(); @@ -158,10 +158,10 @@ clearPostings(attribute::IAttributeVector::EnumHandle eidx, auto updater = [this, &postings](EntryRef posting_idx) -> EntryRef { _postingList.apply(posting_idx, - &postings._additions[0], - &postings._additions[0] + postings._additions.size(), - &postings._removals[0], - &postings._removals[0] + postings._removals.size()); + postings._additions.data(), + postings._additions.data() + postings._additions.size(), + postings._removals.data(), + postings._removals.data() + postings._removals.size()); return posting_idx; }; _dictionary.update_posting_list(er, cmp, updater); @@ -240,11 +240,11 @@ handle_load_posting_lists(LoadedVector& loaded) postings.removeDups(); newIndex = EntryRef(); _postingList.apply(newIndex, - &postings._additions[0], - &postings._additions[0] + + postings._additions.data(), + postings._additions.data() + postings._additions.size(), - &postings._removals[0], - &postings._removals[0] + + postings._removals.data(), + postings._removals.data() + postings._removals.size()); postings.clear(); if (value._docId < docIdLimit) { @@ -262,11 +262,11 @@ handle_load_posting_lists(LoadedVector& loaded) postings.removeDups(); newIndex = EntryRef(); _postingList.apply(newIndex, - &postings._additions[0], - &postings._additions[0] + + postings._additions.data(), + postings._additions.data() + postings._additions.size(), - &postings._removals[0], - &postings._removals[0] + postings._removals.size()); + postings._removals.data(), + postings._removals.data() + postings._removals.size()); similarValues[0]._pidx = newIndex; for (size_t i(0), m(similarValues.size()); i < m; i++) { loaded.write(similarValues[i]); diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp index c17627a5026..d8426ce1a45 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp @@ -20,8 +20,8 @@ PostingListSearchContext(const IEnumStoreDictionary& dictionary, const ISearchContext &baseSearchCtx) : _dictionary(dictionary), _frozenDictionary(_dictionary.get_has_btree_dictionary() ? _dictionary.get_posting_dictionary().getFrozenView() : FrozenDictionary()), - _lowerDictItr(BTreeNode::Ref(), _frozenDictionary.getAllocator()), - _upperDictItr(BTreeNode::Ref(), _frozenDictionary.getAllocator()), + _lowerDictItr(_dictionary.get_has_btree_dictionary() ? DictionaryConstIterator(BTreeNode::Ref(), _frozenDictionary.getAllocator()) : DictionaryConstIterator()), + _upperDictItr(_dictionary.get_has_btree_dictionary() ? DictionaryConstIterator(BTreeNode::Ref(), _frozenDictionary.getAllocator()) : DictionaryConstIterator()), _uniqueValues(0u), _docIdLimit(docIdLimit), _dictSize(_frozenDictionary.size()), diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp index 5ac506e4fc2..b60250256f4 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp @@ -98,8 +98,7 @@ EncodeContext64EBase<false>::writeBits(uint64_t data, uint32_t length) // Shift new bits into cacheInt _cacheInt |= (data << (64 - _cacheFree)); *_valI++ = bswap(_cacheInt); - - data >>= _cacheFree; + data = (_cacheFree < 64) ? data >> _cacheFree : 0; // Initialize variables for receiving new bits length -= _cacheFree; _cacheInt = 0; @@ -194,9 +193,9 @@ writeBits(const uint64_t *bits, uint32_t bitOffset, uint32_t bitLength) if (bitOffset + bitLength < 64) { uint32_t length = bitLength; if (bigEndian) { - uint64_t data = (EC::bswap(*bits) >> - (64 - bitOffset - length)) & - CodingTables::_intMask64[length]; + uint64_t data = ((bitOffset + length) > 0) + ? (EC::bswap(*bits) >> (64 - bitOffset - length)) & CodingTables::_intMask64[length] + : 0; UC64BE_WRITEBITS_NS(o, EC); } else { uint64_t data = (EC::bswap(*bits) >> bitOffset) & diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h index 45005d499fb..74231638213 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h @@ -165,8 +165,7 @@ public: #define UC64BE_DECODEEXPGOLOMB(val, valI, preRead, cacheInt, k, EC) \ do { \ - length = \ - 63 - ::search::bitcompression::EncodeContext64BE::asmlog2(val); \ + length = __builtin_clzl(val); \ unsigned int olength = length; \ val <<= length; \ if (__builtin_expect(length * 2 + 1 + (k) > 64, false)) { \ @@ -174,8 +173,9 @@ public: length = 0; \ } \ val64 = (val >> (63 - olength - (k))) - (UINT64_C(1) << (k)); \ - val <<= olength + 1 + (k); \ - if (__builtin_expect(olength + 1 + (k) == 64, false)) { \ + if (__builtin_expect(olength + 1 + (k) != 64, true)) { \ + val <<= olength + 1 + (k); \ + } else { \ val = 0; \ } \ length += olength + 1 + (k); \ @@ -193,8 +193,7 @@ public: #define UC64BE_DECODEEXPGOLOMB_SMALL(val, valI, preRead, cacheInt, k, \ EC) \ do { \ - length = \ - 63 - ::search::bitcompression::EncodeContext64BE::asmlog2(val); \ + length = __builtin_clzl(val); \ val <<= length; \ val64 = (val >> (63 - length - (k))) - (UINT64_C(1) << (k)); \ val <<= length + 1 + (k); \ @@ -219,8 +218,7 @@ public: #define UC64BE_DECODEEXPGOLOMB_SMALL_APPLY(val, valI, preRead, cacheInt, \ k, EC, resop) \ do { \ - length = \ - 63 - ::search::bitcompression::EncodeContext64BE::asmlog2(val); \ + length = __builtin_clzl(val); \ val <<= length; \ resop (val >> (63 - length - (k))) - (UINT64_C(1) << (k)); \ val <<= length + 1 + (k); \ @@ -231,16 +229,16 @@ public: #define UC64BE_SKIPEXPGOLOMB(val, valI, preRead, cacheInt, k, EC) \ do { \ - length = \ - 63 - ::search::bitcompression::EncodeContext64BE::asmlog2(val); \ + length = __builtin_clzl(val); \ unsigned int olength = length; \ val <<= length; \ if (__builtin_expect(length * 2 + 1 + (k) > 64, false)) { \ UC64BE_READBITS(val, valI, preRead, cacheInt, EC); \ length = 0; \ } \ - val <<= olength + 1 + (k); \ - if (__builtin_expect(olength + 1 + (k) == 64, false)) { \ + if (__builtin_expect(olength + 1 + (k) != 64, true)) { \ + val <<= olength + 1 + (k); \ + } else { \ val = 0; \ } \ length += olength + 1 + (k); \ @@ -258,8 +256,7 @@ public: #define UC64BE_SKIPEXPGOLOMB_SMALL(val, valI, preRead, cacheInt, k, \ EC) \ do { \ - length = \ - 63 - ::search::bitcompression::EncodeContext64BE::asmlog2(val); \ + length = __builtin_clzl(val); \ val <<= length; \ val <<= length + 1 + (k); \ length += length + 1 + (k); \ @@ -394,11 +391,11 @@ public: #define UC64LE_DECODEEXPGOLOMB(val, valI, preRead, cacheInt, k, EC) \ do { \ - unsigned int olength = \ - ::search::bitcompression::EncodeContext64LE::ffsl(val); \ + unsigned int olength = __builtin_ctzl(val); \ length = olength + 1; \ - val >>= length; \ - if (__builtin_expect(length == 64, false)) { \ + if (__builtin_expect(length != 64, true)) { \ + val >>= length; \ + } else { \ val = 0; \ } \ if (__builtin_expect(olength * 2 + 1 + (k) > 64, false)) { \ @@ -423,7 +420,7 @@ public: #define UC64LE_DECODEEXPGOLOMB_SMALL(val, valI, preRead, cacheInt, k, \ EC) \ do { \ - length = ::search::bitcompression::EncodeContext64LE::ffsl(val); \ + length = __builtin_ctzl(val); \ val >>= length + 1; \ val64 = (val & ((UINT64_C(1) << (length + (k))) - 1)) + \ (UINT64_C(1) << (length + (k))) - (UINT64_C(1) << (k)); \ @@ -449,7 +446,7 @@ public: #define UC64LE_DECODEEXPGOLOMB_SMALL_APPLY(val, valI, preRead, cacheInt, \ k, EC, resop) \ do { \ - length = ::search::bitcompression::EncodeContext64LE::ffsl(val); \ + length = __builtin_ctzl(val); \ val >>= length + 1; \ resop (val & ((UINT64_C(1) << (length + (k))) - 1)) + \ (UINT64_C(1) << (length + (k))) - (UINT64_C(1) << (k)); \ @@ -461,11 +458,11 @@ public: #define UC64LE_SKIPEXPGOLOMB(val, valI, preRead, cacheInt, k, EC) \ do { \ - unsigned int olength = \ - ::search::bitcompression::EncodeContext64LE::ffsl(val); \ + unsigned int olength = __builtin_ctzl(val); \ length = olength + 1; \ - val >>= length; \ - if (__builtin_expect(length == 64, false)) { \ + if (__builtin_expect(length != 64, true)) { \ + val >>= length; \ + } else { \ val = 0; \ } \ if (__builtin_expect(olength * 2 + 1 + (k) > 64, false)) { \ @@ -488,7 +485,7 @@ public: #define UC64LE_SKIPEXPGOLOMB_SMALL(val, valI, preRead, cacheInt, k, \ EC) \ do { \ - length = ::search::bitcompression::EncodeContext64LE::ffsl(val); \ + length = __builtin_ctzl(val); \ val >>= length + 1; \ val >>= length + (k); \ length += length + 1 + (k); \ @@ -507,7 +504,11 @@ public: if (length >= cacheFree) { \ cacheInt |= (data << (64 - cacheFree)); \ *bufI++ = EC::bswap(cacheInt); \ - data >>= cacheFree; \ + if (__builtin_expect(cacheFree != 64, true)) { \ + data >>= cacheFree; \ + } else { \ + data = 0; \ + } \ length -= cacheFree; \ cacheInt = 0; \ cacheFree = 64; \ diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp index 9a711a028fb..b0a201d913e 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp @@ -365,7 +365,7 @@ PageDict4SPWriter::flushPage() _prevL3Size - wordsSize * 8; e.padBits(padding); if (wordsSize > 0) { - e.writeBytes(vespalib::ConstArrayRef<char>(&_words[0], wordsSize)); + e.writeBytes(vespalib::ConstArrayRef<char>(_words.data(), wordsSize)); } assert((e.getWriteOffset() & (getPageBitSize() - 1)) == 0); _l6Word = _l3Word; @@ -676,7 +676,7 @@ PageDict4PWriter::flushPage() _countsSize - _countsWordOffset * 8; e.padBits(padding); if (_countsWordOffset > 0) { - e.writeBytes(vespalib::ConstArrayRef(&_words[0], _countsWordOffset)); + e.writeBytes(vespalib::ConstArrayRef(_words.data(), _countsWordOffset)); } assert((e.getWriteOffset() & (getPageBitSize() - 1)) == 0); _l3Word = _pendingCountsWord; @@ -1055,7 +1055,7 @@ lookup(vespalib::stringref key) L7Vector::const_iterator l7lb; l7lb = std::lower_bound(_l7.begin(), _l7.end(), key); - l7Pos = &*l7lb - &_l7[0]; + l7Pos = l7lb - _l7.cbegin(); StartOffset startOffset; uint64_t pageNum = _pFirstPageNum; uint32_t sparsePageNum = _spFirstPageNum; @@ -1863,7 +1863,7 @@ PageDict4Reader::setupPage() uint32_t padding = (getPageBitSize() - wordsSize * 8 - pageOffset) & (getPageBitSize() - 1); _pd.skipBits(padding); _words.resize(wordsSize); - _pd.readBytes(reinterpret_cast<uint8_t *>(&_words[0]), wordsSize); + _pd.readBytes(reinterpret_cast<uint8_t *>(_words.data()), wordsSize); _wc = _words.begin(); _we = _words.end(); checkWordOffsets(_words, _l1SkipChecks, _l2SkipChecks); @@ -1985,7 +1985,7 @@ PageDict4Reader::setupSPage() uint32_t padding = getPageBitSize() - wordsSize * 8 - pageOffset; _spd.skipBits(padding); _spwords.resize(wordsSize); - _spd.readBytes(reinterpret_cast<uint8_t *>(&_spwords[0]), wordsSize); + _spd.readBytes(reinterpret_cast<uint8_t *>(_spwords.data()), wordsSize); _spwc = _spwords.begin(); _spwe = _spwords.end(); checkWordOffsets(_spwords, _l4SkipChecks, _l5SkipChecks); diff --git a/searchlib/src/vespa/searchlib/common/geo_location.cpp b/searchlib/src/vespa/searchlib/common/geo_location.cpp index 1806ba1338c..20408a93a82 100644 --- a/searchlib/src/vespa/searchlib/common/geo_location.cpp +++ b/searchlib/src/vespa/searchlib/common/geo_location.cpp @@ -8,6 +8,12 @@ namespace search::common { namespace { +uint64_t abs_diff(int32_t a, int32_t b) { + return (a > b) + ? (int64_t(a) - int64_t(b)) + : (int64_t(b) - int64_t(a)); +} + ZCurve::BoundingBox to_z(GeoLocation::Box box) { return ZCurve::BoundingBox(box.x.low, box.x.high, box.y.low, box.y.high); @@ -158,13 +164,13 @@ GeoLocation::GeoLocation(Box b, Point p, uint32_t r, Aspect xa) uint64_t GeoLocation::sq_distance_to(Point p) const { if (has_point) { - uint64_t dx = (p.x > point.x) ? (p.x - point.x) : (point.x - p.x); + uint64_t dx = abs_diff(p.x, point.x); if (x_aspect.active()) { // x_aspect is a 32-bit fixed-point number in range [0,1] // this implements dx = (dx * x_aspect) dx = (dx * x_aspect.multiplier) >> 32; } - uint64_t dy = (p.y > point.y) ? (p.y - point.y) : (point.y - p.y); + uint64_t dy = abs_diff(p.y, point.y); return dx*dx + dy*dy; } return 0; diff --git a/searchlib/src/vespa/searchlib/common/geo_location.h b/searchlib/src/vespa/searchlib/common/geo_location.h index 07e6fd055cc..09c77037b03 100644 --- a/searchlib/src/vespa/searchlib/common/geo_location.h +++ b/searchlib/src/vespa/searchlib/common/geo_location.h @@ -2,10 +2,10 @@ #pragma once +#include <vespa/vespalib/geo/zcurve.h> #include <string> #include <cstdint> #include <limits> -#include <vespa/vespalib/geo/zcurve.h> namespace search::common { diff --git a/searchlib/src/vespa/searchlib/common/geo_location_parser.cpp b/searchlib/src/vespa/searchlib/common/geo_location_parser.cpp index 8794169b4a6..d829e1b93e4 100644 --- a/searchlib/src/vespa/searchlib/common/geo_location_parser.cpp +++ b/searchlib/src/vespa/searchlib/common/geo_location_parser.cpp @@ -15,7 +15,7 @@ LOG_SETUP(".searchlib.common.geo_location_parser"); namespace { int getInt(const char * &p) { - int val; + uint32_t val; bool isminus; val = 0; isminus = false; diff --git a/searchlib/src/vespa/searchlib/common/geo_location_spec.h b/searchlib/src/vespa/searchlib/common/geo_location_spec.h index ea0104aa058..f1e3671181d 100644 --- a/searchlib/src/vespa/searchlib/common/geo_location_spec.h +++ b/searchlib/src/vespa/searchlib/common/geo_location_spec.h @@ -2,9 +2,9 @@ #pragma once +#include "geo_location.h" #include <string> #include <cstdint> -#include "geo_location.h" namespace search::common { diff --git a/searchlib/src/vespa/searchlib/common/resultset.cpp b/searchlib/src/vespa/searchlib/common/resultset.cpp index 2e1e431ad82..3a88a310fe8 100644 --- a/searchlib/src/vespa/searchlib/common/resultset.cpp +++ b/searchlib/src/vespa/searchlib/common/resultset.cpp @@ -99,7 +99,7 @@ ResultSet::mergeWithBitOverflow(HitRank default_value) void ResultSet::sort(FastS_IResultSorter & sorter, unsigned int ntop) { - sorter.sortResults(&_rankedHitsArray[0], _rankedHitsArray.size(), ntop); + sorter.sortResults(_rankedHitsArray.data(), _rankedHitsArray.size(), ntop); } std::pair<std::unique_ptr<BitVector>, vespalib::Array<RankedHit>> diff --git a/searchlib/src/vespa/searchlib/common/resultset.h b/searchlib/src/vespa/searchlib/common/resultset.h index 6824fc4170d..a4823d2f372 100644 --- a/searchlib/src/vespa/searchlib/common/resultset.h +++ b/searchlib/src/vespa/searchlib/common/resultset.h @@ -26,7 +26,7 @@ public: void allocArray(unsigned int arrayAllocated); void setBitOverflow(std::unique_ptr<BitVector> newBitOverflow); - const RankedHit * getArray() const { return &_rankedHitsArray[0]; } + const RankedHit * getArray() const { return _rankedHitsArray.data(); } RankedHit & operator [](uint32_t i) { return _rankedHitsArray[i]; } void push_back(RankedHit hit) { _rankedHitsArray.push_back_fast(hit); } unsigned int getArrayUsed() const { return _rankedHitsArray.size(); } diff --git a/searchlib/src/vespa/searchlib/common/sortresults.cpp b/searchlib/src/vespa/searchlib/common/sortresults.cpp index f1756712d2c..59a47dd3312 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.cpp +++ b/searchlib/src/vespa/searchlib/common/sortresults.cpp @@ -209,9 +209,9 @@ FastS_SortSpec::realloc(uint32_t n, size_t & variableWidth, uint32_t & available variableWidth *= 2; available += variableWidth * n; dataSize += variableWidth * n; - uint32_t byteUsed = mySortData - &_binarySortData[0]; + uint32_t byteUsed = mySortData - _binarySortData.data(); _binarySortData.resize(dataSize); - return &_binarySortData[0] + byteUsed; + return _binarySortData.data() + byteUsed; } void @@ -237,7 +237,7 @@ FastS_SortSpec::initSortData(const RankedHit *hits, uint32_t n) uint32_t dataSize = (fixedWidth + variableWidth) * n; uint32_t available = dataSize; _binarySortData.resize(dataSize); - uint8_t *mySortData = &_binarySortData[0]; + uint8_t *mySortData = _binarySortData.data(); _sortDataArray.resize(n); @@ -342,7 +342,7 @@ void FastS_SortSpec::copySortData(uint32_t offset, uint32_t n, uint32_t *idx, char *buf) { - const uint8_t * sortData = &_binarySortData[0]; + const uint8_t * sortData = _binarySortData.data(); uint32_t totalLen = 0; for (uint32_t i = offset; i < (offset + n); ++i, ++idx) { const uint8_t * src = sortData + _sortDataArray[i]._idx; @@ -378,7 +378,7 @@ inline int FastS_SortSpec::Compare(const FastS_SortSpec *self, const SortData &a, const SortData &b) { - const uint8_t * ref = &(self->_binarySortData[0]); + const uint8_t * ref = self->_binarySortData.data(); uint32_t len = a._len < b._len ? a._len : b._len; int retval = memcmp(ref + a._idx, ref + b._idx, len); @@ -448,10 +448,10 @@ void FastS_SortSpec::sortResults(RankedHit a[], uint32_t n, uint32_t topn) { initSortData(a, n); - SortData * sortData = &_sortDataArray[0]; + SortData * sortData = _sortDataArray.data(); { Array<uint32_t> radixScratchPad(n, Alloc::alloc(0, MMAP_LIMIT)); - search::radix_sort(SortDataRadix(&_binarySortData[0]), StdSortDataCompare(&_binarySortData[0]), SortDataEof(), 1, sortData, n, &radixScratchPad[0], 0, 96, topn); + search::radix_sort(SortDataRadix(_binarySortData.data()), StdSortDataCompare(_binarySortData.data()), SortDataEof(), 1, sortData, n, radixScratchPad.data(), 0, 96, topn); } for (uint32_t i(0), m(_sortDataArray.size()); i < m; ++i) { a[i]._rankValue = _sortDataArray[i]._rankValue; diff --git a/searchlib/src/vespa/searchlib/diskindex/docidmapper.h b/searchlib/src/vespa/searchlib/diskindex/docidmapper.h index 90bfa658a72..7c6f53720f2 100644 --- a/searchlib/src/vespa/searchlib/diskindex/docidmapper.h +++ b/searchlib/src/vespa/searchlib/diskindex/docidmapper.h @@ -42,9 +42,9 @@ public: { } void setup(const DocIdMapping &mapping) { - _selector = (mapping._selector != nullptr) ? &((*mapping._selector)[0]) : nullptr; + _selector = (mapping._selector != nullptr) ? mapping._selector->data() : nullptr; _docIdLimit = mapping._docIdLimit; - _selectorLimit = (mapping._selector != nullptr) ? (*mapping._selector).size() : 0u; + _selectorLimit = (mapping._selector != nullptr) ? mapping._selector->size() : 0u; _selectorId = mapping._selectorId; } diff --git a/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp b/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp index 860903174bc..d27ab2e7787 100644 --- a/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp @@ -12,12 +12,12 @@ #include <vespa/searchlib/common/i_flush_token.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/util/filekit.h> -#include <vespa/searchlib/util/dirtraverse.h> #include <vespa/searchlib/util/posting_priority_queue_merger.hpp> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/exceptions.h> #include <filesystem> +#include <system_error> #include <vespa/log/log.h> @@ -107,8 +107,9 @@ FieldMerger::clean_tmp_dirs() while (i > 0) { i--; vespalib::string tmpindexpath = createTmpPath(_field_dir, i); - search::DirectoryTraverse dt(tmpindexpath.c_str()); - if (!dt.RemoveTree()) { + std::error_code ec; + std::filesystem::remove_all(std::filesystem::path(tmpindexpath), ec); + if (ec) { LOG(error, "Failed to clean tmpdir %s", tmpindexpath.c_str()); return false; } diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index e142255252c..4fd9d116244 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -8,13 +8,13 @@ #include <vespa/searchlib/common/documentsummary.h> #include <vespa/searchlib/common/i_flush_token.h> #include <vespa/searchlib/index/schemautil.h> -#include <vespa/searchlib/util/dirtraverse.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/error.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/document/util/queue.h> #include <filesystem> +#include <system_error> #include <vespa/log/log.h> @@ -117,8 +117,9 @@ Fusion::merge(vespalib::Executor& shared_executor, std::shared_ptr<IFlushToken> LOG(error, "\"%s\" is not a directory", _fusion_out_index.get_path().c_str()); return false; } - search::DirectoryTraverse dt(_fusion_out_index.get_path().c_str()); - if (!dt.RemoveTree()) { + std::error_code ec; + std::filesystem::remove_all(std::filesystem::path(_fusion_out_index.get_path()), ec); + if (ec) { LOG(error, "Failed to clean directory \"%s\"", _fusion_out_index.get_path().c_str()); return false; } diff --git a/searchlib/src/vespa/searchlib/engine/docsumrequest.h b/searchlib/src/vespa/searchlib/engine/docsumrequest.h index 27fb5b25a96..d4f3a1ec340 100644 --- a/searchlib/src/vespa/searchlib/engine/docsumrequest.h +++ b/searchlib/src/vespa/searchlib/engine/docsumrequest.h @@ -2,7 +2,6 @@ #pragma once -#include "propertiesmap.h" #include "request.h" #include "lazy_source.h" #include <vespa/document/base/globalid.h> diff --git a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp index 77781d583cb..4eaa5b3eb65 100644 --- a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp +++ b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp @@ -135,7 +135,7 @@ ProtoConverter::search_reply_to_proto(const SearchReply &reply, ProtoSearchReply } } } - proto.set_grouping_blob(&reply.groupResult[0], reply.groupResult.size()); + proto.set_grouping_blob(reply.groupResult.data(), reply.groupResult.size()); const auto &slime_trace = reply.propertiesMap.trace().lookup("slime"); proto.set_slime_trace(slime_trace.get().data(), slime_trace.get().size()); if (reply.my_issues) { diff --git a/searchlib/src/vespa/searchlib/expression/integerresultnode.h b/searchlib/src/vespa/searchlib/expression/integerresultnode.h index a7fe86acd97..e63ac783bc8 100644 --- a/searchlib/src/vespa/searchlib/expression/integerresultnode.h +++ b/searchlib/src/vespa/searchlib/expression/integerresultnode.h @@ -28,7 +28,7 @@ public: T bv(static_cast<const IntegerResultNodeT &>(b)._value); return (_value < bv) ? -1 : (_value > bv) ? 1 : 0; } - void add(const ResultNode & b) override { _value += b.getInteger(); } + void add(const ResultNode & b) override { _value = uint64_t(_value) + uint64_t(b.getInteger()); } void negate() override { _value = - _value; } void multiply(const ResultNode & b) override { if constexpr (std::is_same_v<T, bool>) { diff --git a/searchlib/src/vespa/searchlib/fef/objectstore.cpp b/searchlib/src/vespa/searchlib/fef/objectstore.cpp index 2da08e2915d..4cf185ad55e 100644 --- a/searchlib/src/vespa/searchlib/fef/objectstore.cpp +++ b/searchlib/src/vespa/searchlib/fef/objectstore.cpp @@ -2,8 +2,7 @@ #include "objectstore.h" #include <vespa/vespalib/stllike/hash_map.hpp> -namespace search { -namespace fef { +namespace search::fef { ObjectStore::ObjectStore() : _objectMap() @@ -37,4 +36,3 @@ ObjectStore::get(const vespalib::string & key) const } } -} diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp index b851fc50518..8664b0fc14b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp @@ -18,7 +18,7 @@ HitCollector::sortHitsByScore(size_t topn) _scoreOrder.push_back(i); } ShiftBasedRadixSorter<uint32_t, IndirectScoreRadix, IndirectScoreComparator, 56, true>:: - radix_sort(IndirectScoreRadix(&_hits[0]), IndirectScoreComparator(&_hits[0]), &_scoreOrder[0], _scoreOrder.size(), 16, topn); + radix_sort(IndirectScoreRadix(_hits.data()), IndirectScoreComparator(_hits.data()), _scoreOrder.data(), _scoreOrder.size(), 16, topn); _scoreOrder.resize(topn); } } @@ -28,7 +28,7 @@ HitCollector::sortHitsByDocId() { if (_hitsSortOrder != SortOrder::DOC_ID) { ShiftBasedRadixSorter<Hit, DocIdRadix, DocIdComparator, 24>:: - radix_sort(DocIdRadix(), DocIdComparator(), &_hits[0], _hits.size(), 16); + radix_sort(DocIdRadix(), DocIdComparator(), _hits.data(), _hits.size(), 16); _hitsSortOrder = SortOrder::DOC_ID; _scoreOrder.clear(); } @@ -170,7 +170,7 @@ HitCollector::getSortedHitSequence(size_t max_hits) { size_t num_hits = std::min(_hits.size(), max_hits); sortHitsByScore(num_hits); - return SortedHitSequence(&_hits[0], &_scoreOrder[0], num_hits); + return SortedHitSequence(_hits.data(), _scoreOrder.data(), num_hits); } void diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index 8c03800b92a..8aa806b01cd 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -4,7 +4,6 @@ #include "nearest_neighbor_blueprint.h" #include "nearest_neighbor_iterator.h" #include "nns_index_iterator.h" -#include <vespa/eval/eval/fast_value.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/distance_function_factory.h> @@ -13,45 +12,12 @@ LOG_SETUP(".searchlib.queryeval.nearest_neighbor_blueprint"); -using vespalib::eval::CellType; -using vespalib::eval::FastValueBuilderFactory; -using vespalib::eval::TypedCells; using vespalib::eval::Value; -using vespalib::eval::ValueType; namespace search::queryeval { namespace { -template<typename LCT, typename RCT> -std::unique_ptr<Value> -convert_cells(const ValueType &new_type, std::unique_ptr<Value> old_value) -{ - auto old_cells = old_value->cells().typify<LCT>(); - auto builder = FastValueBuilderFactory::get().create_value_builder<RCT>(new_type); - auto new_cells = builder->add_subspace(); - assert(old_cells.size() == new_cells.size()); - auto p = new_cells.begin(); - for (LCT value : old_cells) { - RCT conv(value); - *p++ = conv; - } - return builder->build(std::move(builder)); -} - -struct ConvertCellsSelector -{ - template <typename LCT, typename RCT> - static auto invoke(const ValueType &new_type, std::unique_ptr<Value> old_value) { - return convert_cells<LCT, RCT>(new_type, std::move(old_value)); - } - auto operator() (CellType from, CellType to, std::unique_ptr<Value> old_value) const { - using MyTypify = vespalib::eval::TypifyCellType; - ValueType new_type = old_value->type().cell_cast(to); - return vespalib::typify_invoke<2,MyTypify,ConvertCellsSelector>(from, to, new_type, std::move(old_value)); - } -}; - vespalib::string to_string(NearestNeighborBlueprint::Algorithm algorithm) { @@ -78,7 +44,8 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f double global_filter_upper_limit) : ComplexLeafBlueprint(field), _attr_tensor(attr_tensor), - _query_tensor(std::move(query_tensor)), + _distance_calc(_attr_tensor, std::move(query_tensor)), + _query_tensor(_distance_calc.query_tensor()), _target_hits(target_hits), _adjusted_target_hits(target_hits), _approximate(approximate), @@ -86,7 +53,6 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _distance_threshold(std::numeric_limits<double>::max()), _global_filter_lower_limit(global_filter_lower_limit), _global_filter_upper_limit(global_filter_upper_limit), - _fallback_dist_fun(), _distance_heap(target_hits), _found_hits(), _algorithm(Algorithm::EXACT), @@ -95,27 +61,13 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _global_filter_hits(), _global_filter_hit_ratio() { - CellType attr_ct = _attr_tensor.getTensorType().cell_type(); - _fallback_dist_fun = search::tensor::make_distance_function(_attr_tensor.distance_metric(), attr_ct); - _dist_fun = _fallback_dist_fun.get(); - assert(_dist_fun); - auto nns_index = _attr_tensor.nearest_neighbor_index(); - if (nns_index) { - _dist_fun = nns_index->distance_function(); - assert(_dist_fun); - } - auto query_ct = _query_tensor->cells().type; - CellType required_ct = _dist_fun->expected_cell_type(); - if (query_ct != required_ct) { - ConvertCellsSelector converter; - _query_tensor = converter(query_ct, required_ct, std::move(_query_tensor)); - } if (distance_threshold < std::numeric_limits<double>::max()) { - _distance_threshold = _dist_fun->convert_threshold(distance_threshold); + _distance_threshold = _distance_calc.function().convert_threshold(distance_threshold); _distance_heap.set_distance_threshold(_distance_threshold); } uint32_t est_hits = _attr_tensor.get_num_docs(); setEstimate(HitEstimate(est_hits, false)); + auto nns_index = _attr_tensor.nearest_neighbor_index(); set_want_global_filter(nns_index && _approximate); } @@ -155,7 +107,7 @@ NearestNeighborBlueprint::set_global_filter(const GlobalFilter &global_filter, d void NearestNeighborBlueprint::perform_top_k(const search::tensor::NearestNeighborIndex* nns_index) { - auto lhs = _query_tensor->cells(); + auto lhs = _query_tensor.cells(); uint32_t k = _adjusted_target_hits; if (_global_filter->has_filter()) { auto filter = _global_filter->filter(); @@ -175,13 +127,12 @@ NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchData switch (_algorithm) { case Algorithm::INDEX_TOP_K_WITH_FILTER: case Algorithm::INDEX_TOP_K: - return NnsIndexIterator::create(tfmd, _found_hits, _dist_fun); + return NnsIndexIterator::create(tfmd, _found_hits, _distance_calc.function()); default: ; } - const Value &qT = *_query_tensor; - return NearestNeighborIterator::create(strict, tfmd, qT, _attr_tensor, - _distance_heap, _global_filter->filter(), _dist_fun); + return NearestNeighborIterator::create(strict, tfmd, _distance_calc, + _distance_heap, _global_filter->filter()); } void @@ -189,7 +140,7 @@ NearestNeighborBlueprint::visitMembers(vespalib::ObjectVisitor& visitor) const { ComplexLeafBlueprint::visitMembers(visitor); visitor.visitString("attribute_tensor", _attr_tensor.getTensorType().to_spec()); - visitor.visitString("query_tensor", _query_tensor->type().to_spec()); + visitor.visitString("query_tensor", _query_tensor.type().to_spec()); visitor.visitInt("target_hits", _target_hits); visitor.visitInt("adjusted_target_hits", _adjusted_target_hits); visitor.visitInt("explore_additional_hits", _explore_additional_hits); diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h index 16b0e13014e..3be7d7fd01f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h @@ -3,6 +3,7 @@ #include "blueprint.h" #include "nearest_neighbor_distance_heap.h" +#include <vespa/searchlib/tensor/distance_calculator.h> #include <vespa/searchlib/tensor/distance_function.h> #include <vespa/searchlib/tensor/nearest_neighbor_index.h> #include <optional> @@ -28,7 +29,8 @@ public: }; private: const tensor::ITensorAttribute& _attr_tensor; - std::unique_ptr<vespalib::eval::Value> _query_tensor; + search::tensor::DistanceCalculator _distance_calc; + const vespalib::eval::Value& _query_tensor; uint32_t _target_hits; uint32_t _adjusted_target_hits; bool _approximate; @@ -36,8 +38,6 @@ private: double _distance_threshold; double _global_filter_lower_limit; double _global_filter_upper_limit; - search::tensor::DistanceFunction::UP _fallback_dist_fun; - const search::tensor::DistanceFunction *_dist_fun; mutable NearestNeighborDistanceHeap _distance_heap; std::vector<search::tensor::NearestNeighborIndex::Neighbor> _found_hits; Algorithm _algorithm; @@ -59,7 +59,7 @@ public: NearestNeighborBlueprint& operator=(const NearestNeighborBlueprint&) = delete; ~NearestNeighborBlueprint(); const tensor::ITensorAttribute& get_attribute_tensor() const { return _attr_tensor; } - const vespalib::eval::Value& get_query_tensor() const { return *_query_tensor; } + const vespalib::eval::Value& get_query_tensor() const { return _query_tensor; } uint32_t get_target_hits() const { return _target_hits; } uint32_t get_adjusted_target_hits() const { return _adjusted_target_hits; } void set_global_filter(const GlobalFilter &global_filter, double estimated_hit_ratio) override; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp index 6a00568bd06..e06fcc614d8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp @@ -2,6 +2,8 @@ #include "nearest_neighbor_iterator.h" #include <vespa/searchlib/common/bitvector.h> +#include <vespa/searchlib/tensor/distance_calculator.h> +#include <vespa/searchlib/tensor/distance_function.h> using search::tensor::ITensorAttribute; using vespalib::ConstArrayRef; @@ -34,11 +36,10 @@ public: NearestNeighborImpl(Params params_in) : NearestNeighborIterator(params_in), - _lhs(params().queryTensor.cells()), _lastScore(0.0) { - assert(is_compatible(params().tensorAttribute.getTensorType(), - params().queryTensor.type())); + assert(is_compatible(params().distance_calc.attribute_tensor().getTensorType(), + params().distance_calc.query_tensor().type())); } ~NearestNeighborImpl(); @@ -64,7 +65,7 @@ public: } void doUnpack(uint32_t docId) override { - double score = params().distanceFunction->to_rawscore(_lastScore); + double score = params().distance_calc.function().to_rawscore(_lastScore); params().tfmd.setRawScore(docId, score); params().distanceHeap.used(_lastScore); } @@ -73,11 +74,9 @@ public: private: double computeDistance(uint32_t docId, double limit) { - auto rhs = params().tensorAttribute.extract_cells_ref(docId); - return params().distanceFunction->calc_with_limit(_lhs, rhs, limit); + return params().distance_calc.calc_with_limit(docId, limit); } - TypedCells _lhs; double _lastScore; }; @@ -105,14 +104,12 @@ std::unique_ptr<NearestNeighborIterator> NearestNeighborIterator::create( bool strict, fef::TermFieldMatchData &tfmd, - const vespalib::eval::Value &queryTensor, - const search::tensor::ITensorAttribute &tensorAttribute, + const search::tensor::DistanceCalculator &distance_calc, NearestNeighborDistanceHeap &distanceHeap, - const search::BitVector *filter, - const search::tensor::DistanceFunction *dist_fun) + const search::BitVector *filter) { - Params params(tfmd, queryTensor, tensorAttribute, distanceHeap, filter, dist_fun); + Params params(tfmd, distance_calc, distanceHeap, filter); if (filter) { return resolve_strict<true>(strict, params); } else { diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h index 66622288d84..0d8f70d15c2 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h @@ -7,10 +7,11 @@ #include <vespa/eval/eval/value.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/tensor/i_tensor_attribute.h> -#include <vespa/searchlib/tensor/distance_function.h> #include <vespa/vespalib/util/priority_queue.h> #include <cmath> +namespace search::tensor { class DistanceCalculator; } + namespace search::queryeval { class NearestNeighborIterator : public SearchIterator @@ -21,24 +22,18 @@ public: struct Params { fef::TermFieldMatchData &tfmd; - const Value &queryTensor; - const ITensorAttribute &tensorAttribute; + const search::tensor::DistanceCalculator &distance_calc; NearestNeighborDistanceHeap &distanceHeap; const search::BitVector *filter; - const search::tensor::DistanceFunction *distanceFunction; - + Params(fef::TermFieldMatchData &tfmd_in, - const Value &queryTensor_in, - const ITensorAttribute &tensorAttribute_in, + const search::tensor::DistanceCalculator &distance_calc_in, NearestNeighborDistanceHeap &distanceHeap_in, - const search::BitVector *filter_in, - const search::tensor::DistanceFunction *distanceFunction_in) + const search::BitVector *filter_in) : tfmd(tfmd_in), - queryTensor(queryTensor_in), - tensorAttribute(tensorAttribute_in), + distance_calc(distance_calc_in), distanceHeap(distanceHeap_in), - filter(filter_in), - distanceFunction(distanceFunction_in) + filter(filter_in) {} }; @@ -49,11 +44,9 @@ public: static std::unique_ptr<NearestNeighborIterator> create( bool strict, fef::TermFieldMatchData &tfmd, - const Value &queryTensor, - const search::tensor::ITensorAttribute &tensorAttribute, + const search::tensor::DistanceCalculator &distance_calc, NearestNeighborDistanceHeap &distanceHeap, - const search::BitVector *filter, - const search::tensor::DistanceFunction *dist_fun); + const search::BitVector *filter); const Params& params() const { return _params; } private: diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp index cd65f01025b..95264a79431 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp @@ -18,13 +18,13 @@ class NeighborVectorIterator : public NnsIndexIterator private: fef::TermFieldMatchData &_tfmd; const std::vector<Neighbor> &_hits; - const search::tensor::DistanceFunction * const _dist_fun; + const search::tensor::DistanceFunction &_dist_fun; uint32_t _idx; double _last_abstract_dist; public: NeighborVectorIterator(fef::TermFieldMatchData &tfmd, const std::vector<Neighbor> &hits, - const search::tensor::DistanceFunction *dist_fun) + const search::tensor::DistanceFunction &dist_fun) : _tfmd(tfmd), _hits(hits), _dist_fun(dist_fun), @@ -54,7 +54,7 @@ public: } void doUnpack(uint32_t docId) override { - double score = _dist_fun->to_rawscore(_last_abstract_dist); + double score = _dist_fun.to_rawscore(_last_abstract_dist); _tfmd.setRawScore(docId, score); } @@ -65,7 +65,7 @@ std::unique_ptr<NnsIndexIterator> NnsIndexIterator::create( fef::TermFieldMatchData &tfmd, const std::vector<Neighbor> &hits, - const search::tensor::DistanceFunction *dist_fun) + const search::tensor::DistanceFunction &dist_fun) { return std::make_unique<NeighborVectorIterator>(tfmd, hits, dist_fun); } diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h index 019ac8579bd..031a603de49 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h @@ -16,7 +16,7 @@ public: static std::unique_ptr<NnsIndexIterator> create( fef::TermFieldMatchData &tfmd, const std::vector<Hit> &hits, - const search::tensor::DistanceFunction *dist_fun); + const search::tensor::DistanceFunction &dist_fun); }; } // namespace diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index ae34cdd66c8..9e0ccb8d37a 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -11,6 +11,7 @@ vespa_add_library(searchlib_tensor OBJECT direct_tensor_attribute.cpp direct_tensor_saver.cpp direct_tensor_store.cpp + distance_calculator.cpp distance_function_factory.cpp euclidean_distance.cpp geo_degrees_distance.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 2fdb73fcf96..c713b3ef335 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -8,9 +8,9 @@ #include "tensor_attribute.hpp" #include <vespa/eval/eval/value.h> #include <vespa/fastlib/io/bufferedfile.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/load_utils.h> #include <vespa/searchlib/attribute/readerbase.h> -#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/lambdatask.h> @@ -102,10 +102,16 @@ BlobSequenceReader::is_present() { } +bool +DenseTensorAttribute::tensor_is_unchanged(DocId docid, const vespalib::eval::Value& new_tensor) const +{ + auto old_tensor = extract_cells_ref(docid); + return _comp.equals(old_tensor, new_tensor.cells()); +} + void DenseTensorAttribute::internal_set_tensor(DocId docid, const vespalib::eval::Value& tensor) { - checkTensorType(tensor); consider_remove_from_index(docid); EntryRef ref = _denseTensorStore.setTensor(tensor); setTensorRef(docid, ref); @@ -152,7 +158,8 @@ DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, con const NearestNeighborIndexFactory& index_factory) : TensorAttribute(baseFileName, cfg, _denseTensorStore), _denseTensorStore(cfg.tensorType(), get_memory_allocator()), - _index() + _index(), + _comp(cfg.tensorType()) { if (cfg.hnsw_index_params().has_value()) { auto tensor_type = cfg.tensorType(); @@ -180,6 +187,7 @@ DenseTensorAttribute::clearDoc(DocId docId) void DenseTensorAttribute::setTensor(DocId docId, const vespalib::eval::Value &tensor) { + checkTensorType(tensor); internal_set_tensor(docId, tensor); if (_index) { _index->add_document(docId); @@ -189,16 +197,26 @@ DenseTensorAttribute::setTensor(DocId docId, const vespalib::eval::Value &tensor std::unique_ptr<PrepareResult> DenseTensorAttribute::prepare_set_tensor(DocId docid, const vespalib::eval::Value& tensor) const { + checkTensorType(tensor); if (_index) { + if (tensor_is_unchanged(docid, tensor)) { + // Don't make changes to the nearest neighbor index when the inserted tensor is unchanged. + // With this optimization we avoid doing unnecessary costly work, first removing the vector point, then inserting the same point. + return {}; + } return _index->prepare_add_document(docid, tensor.cells(), getGenerationHandler().takeGuard()); } - return std::unique_ptr<PrepareResult>(); + return {}; } void DenseTensorAttribute::complete_set_tensor(DocId docid, const vespalib::eval::Value& tensor, std::unique_ptr<PrepareResult> prepare_result) { + if (_index && !prepare_result) { + // The tensor is unchanged. + return; + } internal_set_tensor(docid, tensor); if (_index) { _index->complete_add_document(docid, std::move(prepare_result)); diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h index da7a88af1be..1138a4f4433 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -6,6 +6,7 @@ #include "dense_tensor_store.h" #include "doc_vector_access.h" #include "tensor_attribute.h" +#include "typed_cells_comparator.h" #include <memory> namespace search::tensor { @@ -20,7 +21,9 @@ class DenseTensorAttribute : public TensorAttribute, public DocVectorAccess { private: DenseTensorStore _denseTensorStore; std::unique_ptr<NearestNeighborIndex> _index; + TypedCellsComparator _comp; + bool tensor_is_unchanged(DocId docid, const vespalib::eval::Value& new_tensor) const; void internal_set_tensor(DocId docid, const vespalib::eval::Value& tensor); void consider_remove_from_index(DocId docid); vespalib::MemoryUsage update_stat() override; diff --git a/searchlib/src/vespa/searchlib/tensor/distance_calculator.cpp b/searchlib/src/vespa/searchlib/tensor/distance_calculator.cpp new file mode 100644 index 00000000000..c53d50bc9ff --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/distance_calculator.cpp @@ -0,0 +1,90 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "distance_calculator.h" +#include "distance_function_factory.h" +#include "nearest_neighbor_index.h" +#include <vespa/eval/eval/fast_value.h> + +using vespalib::eval::CellType; +using vespalib::eval::FastValueBuilderFactory; +using vespalib::eval::TypedCells; +using vespalib::eval::Value; +using vespalib::eval::ValueType; + +namespace { + +template<typename LCT, typename RCT> +std::unique_ptr<Value> +convert_cells(const ValueType& new_type, std::unique_ptr<Value> old_value) +{ + auto old_cells = old_value->cells().typify<LCT>(); + auto builder = FastValueBuilderFactory::get().create_value_builder<RCT>(new_type); + auto new_cells = builder->add_subspace(); + assert(old_cells.size() == new_cells.size()); + auto p = new_cells.begin(); + for (LCT value : old_cells) { + RCT conv(value); + *p++ = conv; + } + return builder->build(std::move(builder)); +} + +struct ConvertCellsSelector +{ + template <typename LCT, typename RCT> + static auto invoke(const ValueType& new_type, std::unique_ptr<Value> old_value) { + return convert_cells<LCT, RCT>(new_type, std::move(old_value)); + } + auto operator() (CellType from, CellType to, std::unique_ptr<Value> old_value) const { + using MyTypify = vespalib::eval::TypifyCellType; + ValueType new_type = old_value->type().cell_cast(to); + return vespalib::typify_invoke<2,MyTypify,ConvertCellsSelector>(from, to, new_type, std::move(old_value)); + } +}; + +} + +namespace search::tensor { + +DistanceCalculator::DistanceCalculator(const tensor::ITensorAttribute& attr_tensor, + std::unique_ptr<vespalib::eval::Value> query_tensor_in) + : _attr_tensor(attr_tensor), + _query_tensor_uptr(std::move(query_tensor_in)), + _query_tensor(), + _query_tensor_cells(), + _dist_fun_uptr(make_distance_function(_attr_tensor.distance_metric(), + _attr_tensor.getTensorType().cell_type())), + _dist_fun(_dist_fun_uptr.get()) +{ + assert(_dist_fun); + auto nns_index = _attr_tensor.nearest_neighbor_index(); + if (nns_index) { + _dist_fun = nns_index->distance_function(); + assert(_dist_fun); + } + auto query_ct = _query_tensor_uptr->cells().type; + CellType required_ct = _dist_fun->expected_cell_type(); + if (query_ct != required_ct) { + ConvertCellsSelector converter; + _query_tensor_uptr = converter(query_ct, required_ct, std::move(_query_tensor_uptr)); + } + _query_tensor = _query_tensor_uptr.get(); + _query_tensor_cells = _query_tensor->cells(); +} + +DistanceCalculator::DistanceCalculator(const tensor::ITensorAttribute& attr_tensor, + const vespalib::eval::Value& query_tensor_in, + const DistanceFunction& function_in) + : _attr_tensor(attr_tensor), + _query_tensor_uptr(), + _query_tensor(&query_tensor_in), + _query_tensor_cells(_query_tensor->cells()), + _dist_fun_uptr(), + _dist_fun(&function_in) +{ +} + +DistanceCalculator::~DistanceCalculator() = default; + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h new file mode 100644 index 00000000000..eeb66887598 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "distance_function.h" +#include "i_tensor_attribute.h" + +namespace vespalib::eval { struct Value; } + +namespace search::tensor { + +/** + * Class used to calculate the distance between two n-dimensional vectors, + * where one is stored in a TensorAttribute and the other comes from the query. + * + * The distance function to use is defined in the TensorAttribute. + */ +class DistanceCalculator { +private: + const tensor::ITensorAttribute& _attr_tensor; + std::unique_ptr<vespalib::eval::Value> _query_tensor_uptr; + const vespalib::eval::Value* _query_tensor; + vespalib::eval::TypedCells _query_tensor_cells; + std::unique_ptr<DistanceFunction> _dist_fun_uptr; + const DistanceFunction* _dist_fun; + +public: + DistanceCalculator(const tensor::ITensorAttribute& attr_tensor, + std::unique_ptr<vespalib::eval::Value> query_tensor_in); + + /** + * Only used by unit tests where ownership of query tensor and distance function is handled outside. + */ + DistanceCalculator(const tensor::ITensorAttribute& attr_tensor, + const vespalib::eval::Value& query_tensor_in, + const DistanceFunction& function_in); + + ~DistanceCalculator(); + + const tensor::ITensorAttribute& attribute_tensor() const { return _attr_tensor; } + const vespalib::eval::Value& query_tensor() const { return *_query_tensor; } + const DistanceFunction& function() const { return *_dist_fun; } + + double calc_with_limit(uint32_t docid, double limit) const { + return _dist_fun->calc_with_limit(_query_tensor_cells, _attr_tensor.extract_cells_ref(docid), limit); + } +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function.h b/searchlib/src/vespa/searchlib/tensor/distance_function.h index 77873cb7ced..d5ebf656189 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_function.h @@ -23,7 +23,7 @@ public: DistanceFunction(vespalib::eval::CellType expected) : _expect_cell_type(expected) {} - virtual ~DistanceFunction() {} + virtual ~DistanceFunction() = default; // input (query) vectors must be converted to this cell type: vespalib::eval::CellType expected_cell_type() const { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index e82f31df38e..2ee1b268449 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -436,7 +436,7 @@ HnswIndex::prepare_add_document(uint32_t docid, if (max_nodes < _cfg.min_size_before_two_phase()) { // the first documents added will do all work in write thread // to ensure they are linked together: - return std::unique_ptr<PrepareResult>(); + return std::make_unique<PreparedFirstAddDoc>(); } PreparedAddDoc op = internal_prepare_add(docid, vector, std::move(read_guard)); return std::make_unique<PreparedAddDoc>(std::move(op)); diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 72a10724ff1..3f5a9d514ed 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -152,6 +152,8 @@ protected: const BitVector *filter, uint32_t explore_k, double distance_threshold) const; + struct PreparedFirstAddDoc : public PrepareResult {}; + struct PreparedAddDoc : public PrepareResult { using ReadGuard = vespalib::GenerationHandler::Guard; uint32_t docid; diff --git a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp index a668387e5bd..58e625e6aca 100644 --- a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp @@ -68,7 +68,7 @@ struct MyFastValueView final : Value { { const StringIdVector &labels = handle_view; for (size_t i = 0; i < num_spaces; ++i) { - ConstArrayRef<string_id> addr(&labels[i * num_mapped], num_mapped); + ConstArrayRef<string_id> addr(labels.data() + (i * num_mapped), num_mapped); my_index.map.add_mapping(FastAddrMap::hash_labels(addr)); } assert(my_index.map.size() == num_spaces); diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index add5184c4eb..78c58e86a3b 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -135,7 +135,7 @@ TensorAttribute::addDoc(DocId &docId) } void -TensorAttribute::checkTensorType(const vespalib::eval::Value &tensor) +TensorAttribute::checkTensorType(const vespalib::eval::Value &tensor) const { const ValueType &fieldTensorType = getConfig().tensorType(); const ValueType &tensorType = tensor.type(); diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index ae6a4a302ea..c8aa42c6133 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -32,7 +32,7 @@ protected: template <typename RefType> void doCompactWorst(); - void checkTensorType(const vespalib::eval::Value &tensor); + void checkTensorType(const vespalib::eval::Value &tensor) const; void setTensorRef(DocId docId, EntryRef ref); virtual vespalib::MemoryUsage update_stat(); virtual vespalib::MemoryUsage memory_usage() const; diff --git a/searchlib/src/vespa/searchlib/tensor/typed_cells_comparator.h b/searchlib/src/vespa/searchlib/tensor/typed_cells_comparator.h new file mode 100644 index 00000000000..d1c890be961 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/typed_cells_comparator.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/cell_type.h> +#include <vespa/eval/eval/typed_cells.h> +#include <vespa/eval/eval/value_type.h> +#include <cstring> + +namespace search::tensor { + +/** + * Comparator used to compare two vespalib::eval::TypedCells instances. + * + * The caller must first validate that they are of the same vespalib::eval::ValueType. + */ +class TypedCellsComparator { +private: + size_t _mem_size; + +public: + TypedCellsComparator(const vespalib::eval::ValueType& type) + : _mem_size(vespalib::eval::CellTypeUtils::mem_size(type.cell_type(), type.dense_subspace_size())) + {} + bool equals(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs) const { + return std::memcmp(lhs.data, rhs.data, _mem_size) == 0; + } +}; + +} diff --git a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.cpp b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.cpp index d240e5a7c6c..d1bb464fc37 100644 --- a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.cpp +++ b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.cpp @@ -10,10 +10,7 @@ namespace search { namespace { struct MockReadGuard : public IDocumentMetaStoreContext::IReadGuard { - virtual const search::IDocumentMetaStore &get() const override { - search::IDocumentMetaStore *nullStore = nullptr; - return static_cast<search::IDocumentMetaStore &>(*nullStore); - } + virtual const search::IDocumentMetaStore &get() const override { abort(); } }; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.cpp b/searchlib/src/vespa/searchlib/transactionlog/common.cpp index 4130ad0bc06..d4192fe0beb 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/common.cpp @@ -3,7 +3,9 @@ #include "common.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/fastos/file.h> +#include <filesystem> #include <stdexcept> +#include <system_error> namespace search::transactionlog { @@ -32,7 +34,9 @@ makeDirectory(const char * dir) if ( FastOS_File::Stat(dir, &st) ) { retval = st._isDirectory ? 0 : -2; } else { - retval = FastOS_File::MakeDirectory(dir) ? 0 : -3; + std::error_code ec; + std::filesystem::create_directory(std::filesystem::path(dir), ec); + retval = (!ec) ? 0 : -3; } return retval; diff --git a/searchlib/src/vespa/searchlib/util/comprfile.cpp b/searchlib/src/vespa/searchlib/util/comprfile.cpp index 61eeca6fc2d..bde246a1239 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.cpp +++ b/searchlib/src/vespa/searchlib/util/comprfile.cpp @@ -155,7 +155,7 @@ ComprFileReadBase::SetPosition(uint64_t newPosition, bool readAll, ComprFileDecodeContext &decodeContext, int &bitOffset, - FastOS_FileInterface &file, + FastOS_FileInterface *file, uint64_t &fileReadByteOffset, uint64_t fileSize, ComprBuffer &cbuf) @@ -176,7 +176,7 @@ ComprFileReadBase::SetPosition(uint64_t newPosition, readAll, decodeContext, bitOffset, - file, + *file, fileReadByteOffset, fileSize, cbuf); @@ -200,7 +200,7 @@ ComprFileReadBase::SetPosition(uint64_t newPosition, readAll, decodeContext, bitOffset, - file, + *file, fileReadByteOffset, fileSize, cbuf); @@ -221,9 +221,8 @@ ComprFileReadBase::SetPosition(uint64_t newPosition, (cbuf.getUnitBitSize() - 1)); assert(pos <= static_cast<int64_t>(fileSize)); - - file.SetPosition(pos); - assert(pos == file.GetPosition()); + file->SetPosition(pos); + assert(pos == file->GetPosition()); decodeContext.emptyBuffer(newPosition); assert(decodeContext.getBitPos(bitOffset, @@ -337,7 +336,7 @@ ComprFileReadContext::setPosition(uint64_t newPosition) _readAll, *_decodeContext, _bitOffset, - *_file, + _file, _fileReadByteOffset, _fileSize, *this); diff --git a/searchlib/src/vespa/searchlib/util/comprfile.h b/searchlib/src/vespa/searchlib/util/comprfile.h index 2ee95a53235..dc8cf6185fc 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.h +++ b/searchlib/src/vespa/searchlib/util/comprfile.h @@ -76,7 +76,7 @@ public: bool readAll, ComprFileDecodeContext &decodeContext, int &bitOffset, - FastOS_FileInterface &file, + FastOS_FileInterface *file, uint64_t &fileReadByteOffset, uint64_t fileSize, ComprBuffer &cbuf); diff --git a/searchlib/src/vespa/searchlib/util/dirtraverse.cpp b/searchlib/src/vespa/searchlib/util/dirtraverse.cpp index 07dbc9a247d..c1e8b6b7396 100644 --- a/searchlib/src/vespa/searchlib/util/dirtraverse.cpp +++ b/searchlib/src/vespa/searchlib/util/dirtraverse.cpp @@ -2,275 +2,63 @@ #include "dirtraverse.h" #include <vespa/vespalib/util/size_literals.h> -#include <vespa/fastos/file.h> -#include <cassert> -#include <cstring> +#include <filesystem> +#include <system_error> namespace search { -extern "C" { -static int cmpname(const void *av, const void *bv) -{ - const DirectoryTraverse::Name *const a = - *(const DirectoryTraverse::Name *const *) av; - const DirectoryTraverse::Name *const b = - *(const DirectoryTraverse::Name *const *) bv; - return a->_name.compare(b->_name.c_str()); -} -} - -DirectoryTraverse::Name::Name(const char *name) - : _name(name), - _next(nullptr) -{ -} -DirectoryTraverse::Name::~Name() = default; - -DirectoryTraverse::Name * -DirectoryTraverse::Name::sort(Name *head, int count) -{ - Name *nl; - Name **names; - int i; - - names = new Name *[count]; - i = 0; - for(nl = head; nl != nullptr; nl = nl->_next) - names[i++] = nl; - assert(i == count); - qsort(names, count, sizeof(Name *), cmpname); - for (i = 0; i < count; i++) { - if (i + 1 < count) - names[i]->_next = names[i + 1]; - else - names[i]->_next = nullptr; - } - head = names[0]; - delete [] names; - return head; -} - - -void -DirectoryTraverse::QueueDir(const char *name) -{ - Name *n = new Name(name); - if (_dirTail == nullptr) - _dirHead = n; - else - _dirTail->_next = n; - _dirTail = n; -} +namespace fs = std::filesystem; +namespace { -void -DirectoryTraverse::PushDir(const char *name) -{ - Name *n = new Name(name); - n->_next = _pdirHead; - _pdirHead = n; -} - - -void -DirectoryTraverse::PushRemoveDir(const char *name) -{ - Name *n = new Name(name); - n->_next = _rdirHead; - _rdirHead = n; -} - - -void -DirectoryTraverse::PushPushedDirs() -{ - Name *n; - while (_pdirHead != nullptr) { - n = _pdirHead; - _pdirHead = n->_next; - n->_next = _dirHead; - _dirHead = n; - if (_dirTail == nullptr) - _dirTail = n; - } -} - - -DirectoryTraverse::Name * -DirectoryTraverse::UnQueueDir() -{ - Name *n; - PushPushedDirs(); - if (_dirHead == nullptr) - return nullptr; - n = _dirHead; - _dirHead = n->_next; - n->_next = nullptr; - if (_dirHead == nullptr) - _dirTail = nullptr; - return n; -} - -DirectoryTraverse::Name * -DirectoryTraverse::UnQueueName() -{ - Name *n; - if (_nameHead == nullptr) - return nullptr; - n = _nameHead; - _nameHead = n->_next; - n->_next = nullptr; - _nameCount--; - return n; -} - - -void -DirectoryTraverse::ScanSingleDir() +uint64_t +try_get_tree_size(const std::string& base_dir) { - assert(_nameHead == nullptr); - assert(_nameCount == 0); - delete _curDir; - _fullDirName.clear(); - _curDir = UnQueueDir(); - if (_curDir == nullptr) - return; - _fullDirName = _baseDir; - if ( ! _curDir->_name.empty()) { - _fullDirName += "/" + _curDir->_name; + fs::path path(base_dir); + std::error_code ec; + fs::recursive_directory_iterator dir_itr(path, fs::directory_options::skip_permission_denied, ec); + if (ec) { + return 0; } - FastOS_DirectoryScan *dirscan = new FastOS_DirectoryScan(_fullDirName.c_str()); - while (dirscan->ReadNext()) { - const char *name = dirscan->GetName(); - if (strcmp(name, ".") == 0 || - strcmp(name, "..") == 0) - continue; - Name *nl = new Name(name); - nl->_next = _nameHead; - _nameHead = nl; - _nameCount++; - } - if (_nameCount > 1) - _nameHead = _nameHead->sort(_nameHead, _nameCount); - delete dirscan; -} - - -bool -DirectoryTraverse::NextName() -{ - delete _curName; - _curName = nullptr; - while (_nameHead == nullptr && (_dirHead != nullptr || _pdirHead != nullptr)) - ScanSingleDir(); - if (_nameHead == nullptr) - return false; - _curName = UnQueueName(); - _fullName = _fullDirName + "/" + _curName->_name; - _relName = _fullName.c_str() + (_baseDir.size() + 1); - return true; -} - - -bool -DirectoryTraverse::NextRemoveDir() -{ - Name *curName; - delete _curName; - _curName = nullptr; - if (_rdirHead == nullptr) - return false; - curName = _rdirHead; - _rdirHead = curName->_next; - _fullName = _baseDir + "/" + curName->_name; - _relName = _fullName.c_str() + _baseDir.size() + 1; - delete curName; - return true; -} - - -bool -DirectoryTraverse::RemoveTree() -{ - FastOS_StatInfo statInfo; - - while (NextName()) { - const char *relname = GetRelName(); - const char *fullname = GetFullName(); - if (FastOS_File::Stat(fullname, &statInfo)) { - if (statInfo._isDirectory) { - PushDir(relname); - PushRemoveDir(relname); - } else { - FastOS_File::Delete(fullname); + uint64_t total_size = 0; + constexpr uint64_t block_size = 4_Ki; + for (const auto &elem : dir_itr) { + if (fs::is_regular_file(elem.path()) && !fs::is_symlink(elem.path())) { + const auto size = elem.file_size(ec); + if (!ec) { + // round up size to file system block size (assumed to be 4 KiB) + auto adj_size = ((size + block_size - 1) / block_size) * block_size; + total_size += adj_size; } } } - while (NextRemoveDir()) { - const char *fullname = GetFullName(); - FastOS_File::RemoveDirectory(fullname); - } - FastOS_File::RemoveDirectory(_baseDir.c_str()); - return true; + return total_size; +} + } uint64_t DirectoryTraverse::GetTreeSize() { - FastOS_StatInfo statInfo; - uint64_t size = 0; - const uint64_t blockSize = 4_Ki; - - while (NextName()) { - const char *relname = GetRelName(); - const char *fullname = GetFullName(); - if (FastOS_File::Stat(fullname, &statInfo)) { - uint64_t adjSize = ((statInfo._size + blockSize - 1) / blockSize) * blockSize; - size += adjSize; - if (statInfo._isDirectory) { - PushDir(relname); - } + // Since try_get_tree_size may throw on concurrent directory + // modifications, immediately retry a bounded number of times if this + // happens. Number of retries chosen randomly by counting fingers. + for (int i = 0; i < 10; ++i) { + try { + return try_get_tree_size(_base_dir); + } catch (const fs::filesystem_error&) { + // Go around for another spin that hopefully won't race. } } - return size; + return 0; } -DirectoryTraverse::DirectoryTraverse(const char *baseDir) - : _baseDir(baseDir), - _nameHead(nullptr), - _nameCount(0), - _dirHead(nullptr), - _dirTail(nullptr), - _pdirHead(nullptr), - _rdirHead(nullptr), - _curDir(nullptr), - _curName(nullptr), - _fullDirName(), - _fullName(), - _relName(nullptr) +DirectoryTraverse::DirectoryTraverse(const std::string& base_dir) + : _base_dir(base_dir) { - QueueDir(""); - ScanSingleDir(); } - -DirectoryTraverse::~DirectoryTraverse() -{ - delete _curDir; - delete _curName; - PushPushedDirs(); - while (_dirHead != nullptr) - delete UnQueueDir(); - while (_nameHead != nullptr) - delete UnQueueName(); - while (_rdirHead != nullptr) { - Name *n; - n = _rdirHead; - _rdirHead = n->_next; - n->_next = nullptr; - delete n; - } -} +DirectoryTraverse::~DirectoryTraverse() = default; } // namespace search diff --git a/searchlib/src/vespa/searchlib/util/dirtraverse.h b/searchlib/src/vespa/searchlib/util/dirtraverse.h index 4a96ad0935d..c26246e2596 100644 --- a/searchlib/src/vespa/searchlib/util/dirtraverse.h +++ b/searchlib/src/vespa/searchlib/util/dirtraverse.h @@ -7,54 +7,16 @@ namespace search { +/* + * Class used to get size of directory tree on disk. + */ class DirectoryTraverse { private: - DirectoryTraverse(const DirectoryTraverse &); - DirectoryTraverse& operator=(const DirectoryTraverse &); - -public: - class Name - { - private: - Name(const Name &); - Name& operator=(const Name &); - - public: - std::string _name; - Name *_next; - explicit Name(const char *name); - ~Name(); - static Name *sort(Name *head, int count); - }; -private: - std::string _baseDir; - Name *_nameHead; - int _nameCount; - Name *_dirHead; - Name *_dirTail; - Name *_pdirHead; - Name *_rdirHead; - Name *_curDir; - Name *_curName; - std::string _fullDirName; - std::string _fullName; - const char *_relName; + std::string _base_dir; public: - const char *GetFullName() const { return _fullName.c_str(); } - const char *GetRelName() const { return _relName; } - void QueueDir(const char *name); - void PushDir(const char *name); - void PushRemoveDir(const char *name); - void PushPushedDirs(); - Name *UnQueueDir(); - Name *UnQueueName(); - void ScanSingleDir(); - bool NextName(); - bool NextRemoveDir(); - bool RemoveTree(); uint64_t GetTreeSize(); // Returns size of directory in bytes - explicit DirectoryTraverse(const char *baseDir); + explicit DirectoryTraverse(const std::string& base_dir); ~DirectoryTraverse(); }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index eb4f1a19e06..d5fdee096b1 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attributedfw.h" -#include "docsumstate.h" #include "docsumwriter.h" #include "docsum_field_writer_state.h" #include <vespa/eval/eval/value.h> @@ -39,11 +38,6 @@ AttrDFW::AttrDFW(const vespalib::string & attrName) : { } -const attribute::IAttributeVector & -AttrDFW::get_attribute(const GetDocsumsState& s) const { - return *s.getAttribute(getIndex()); -} - namespace { class SingleAttrDFW : public AttrDFW @@ -53,14 +47,11 @@ public: AttrDFW(attrName) { } void insertField(uint32_t docid, GetDocsumsState *state, ResType type, Inserter &target) override; - bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const override; + bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const override { + return get_attribute(*state).isUndefined(docid); + } }; -bool SingleAttrDFW::isDefaultValue(uint32_t docid, const GetDocsumsState * state) const -{ - return get_attribute(*state).isUndefined(docid); -} - void SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState * state, ResType type, Inserter &target) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h index cf42dac75c0..35f67fd5446 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h @@ -3,7 +3,7 @@ #pragma once #include "docsumfieldwriter.h" - +#include "docsumstate.h" namespace search { class MatchingElementsFields; } namespace search::attribute { class IAttributeVector; } @@ -27,7 +27,9 @@ class AttrDFW : public ISimpleDFW private: vespalib::string _attrName; protected: - const attribute::IAttributeVector& get_attribute(const GetDocsumsState& s) const; + const attribute::IAttributeVector& get_attribute(const GetDocsumsState& s) const { + return *s.getAttribute(getIndex()); + } const vespalib::string & getAttributeName() const override { return _attrName; } public: AttrDFW(const vespalib::string & attrName); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index f55a13a9604..24642c418fd 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -48,18 +48,13 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & throw IllegalArgumentException("Missing argument"); } } else if (overrideName == "summaryfeatures") { - SummaryFeaturesDFW *fw = new SummaryFeaturesDFW(); - fieldWriter.reset(fw); - fw->init(getEnvironment()); + fieldWriter = std::make_unique<SummaryFeaturesDFW>(getEnvironment()); rc = true; } else if (overrideName == "rankfeatures") { - RankFeaturesDFW * fw = new RankFeaturesDFW(); - fw->init(getEnvironment()); - fieldWriter.reset(fw); + fieldWriter = std::make_unique<RankFeaturesDFW>(getEnvironment()); rc = true; } else if (overrideName == "empty") { - EmptyDFW *fw = new EmptyDFW(); - fieldWriter.reset(fw); + fieldWriter = std::make_unique<EmptyDFW>(); rc = true; } else if (overrideName == "copy") { if ( ! argument.empty() ) { @@ -71,21 +66,18 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & } } else if (overrideName == "absdist") { if (getEnvironment()) { - IAttributeManager *am = getEnvironment()->getAttributeManager(); - fieldWriter = AbsDistanceDFW::create(argument.c_str(), am); - rc = fieldWriter.get(); + fieldWriter = AbsDistanceDFW::create(argument.c_str(), getEnvironment()->getAttributeManager()); + rc = static_cast<bool>(fieldWriter); } } else if (overrideName == "positions") { if (getEnvironment()) { - IAttributeManager *am = getEnvironment()->getAttributeManager(); - fieldWriter = PositionsDFW::create(argument.c_str(), am, resultConfig.useV8geoPositions()); - rc = fieldWriter.get(); + fieldWriter = PositionsDFW::create(argument.c_str(), getEnvironment()->getAttributeManager(), resultConfig.useV8geoPositions()); + rc = static_cast<bool>(fieldWriter); } } else if (overrideName == "geopos") { if (getEnvironment()) { - IAttributeManager *am = getEnvironment()->getAttributeManager(); - fieldWriter = GeoPositionDFW::create(argument.c_str(), am, resultConfig.useV8geoPositions()); - rc = fieldWriter.get(); + fieldWriter = GeoPositionDFW::create(argument.c_str(), getEnvironment()->getAttributeManager(), resultConfig.useV8geoPositions()); + rc = static_cast<bool>(fieldWriter); } } else if (overrideName == "attribute") { if (getEnvironment() && getEnvironment()->getAttributeManager()) { @@ -95,11 +87,12 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & } else if (overrideName == "attributecombiner") { if (getEnvironment() && getEnvironment()->getAttributeManager()) { auto attr_ctx = getEnvironment()->getAttributeManager()->createContext(); - fieldWriter = AttributeCombinerDFW::create(fieldName, *attr_ctx, false, std::shared_ptr<MatchingElementsFields>()); + const string& source_field = argument.empty() ? fieldName : argument; + fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, false, std::shared_ptr<MatchingElementsFields>()); rc = static_cast<bool>(fieldWriter); } } else if (overrideName == "matchedattributeelementsfilter") { - string source_field = argument.empty() ? fieldName : argument; + const string& source_field = argument.empty() ? fieldName : argument; if (getEnvironment() && getEnvironment()->getAttributeManager()) { auto attr_ctx = getEnvironment()->getAttributeManager()->createContext(); if (attr_ctx->getAttribute(source_field) != nullptr) { @@ -110,7 +103,7 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & rc = static_cast<bool>(fieldWriter); } } else if (overrideName == "matchedelementsfilter") { - string source_field = argument.empty() ? fieldName : argument; + const string& source_field = argument.empty() ? fieldName : argument; if (getEnvironment() && getEnvironment()->getAttributeManager()) { auto attr_ctx = getEnvironment()->getAttributeManager()->createContext(); fieldWriter = MatchedElementsFilterDFW::create(source_field, resultConfig.GetFieldNameEnum().Lookup(source_field.c_str()), @@ -135,7 +128,7 @@ DynamicDocsumConfig::configure(const vespa::config::search::SummarymapConfig &cf const vespa::config::search::SummarymapConfig::Override & o = cfg.override[i]; bool rc(false); IDocsumFieldWriter::UP fieldWriter = createFieldWriter(o.field, o.command, o.arguments, rc, matching_elems_fields); - if (rc && fieldWriter.get() != NULL) { + if (rc && fieldWriter) { rc = _writer->Override(o.field.c_str(), fieldWriter.release()); // OBJECT HAND-OVER } if (!rc) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp index 47d94a716f7..10b51e1f8e5 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp @@ -3,9 +3,11 @@ #include "docsumfieldwriter.h" #include "idocsumenvironment.h" #include "docsumstate.h" +#include "summaryfieldconverter.h" #include <vespa/searchlib/common/documentlocations.h> #include <vespa/searchlib/common/location.h> #include <vespa/searchlib/parsequery/stackdumpiterator.h> +#include <vespa/vespalib/data/slime/slime.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.docsumfieldwriter"); @@ -44,7 +46,8 @@ EmptyDFW::insertField(uint32_t, GetDocsumsState *, ResType, vespalib::slime::Ins //-------------------------------------------------------------------------- CopyDFW::CopyDFW() - : _inputFieldEnumValue(static_cast<uint32_t>(-1)) + : _inputFieldEnumValue(static_cast<uint32_t>(-1)), + _input_field_name() { } @@ -54,6 +57,7 @@ bool CopyDFW::Init(const ResultConfig & config, const char *inputField) { _inputFieldEnumValue = config.GetFieldNameEnum().Lookup(inputField); + _input_field_name = inputField; if (_inputFieldEnumValue >= config.GetFieldNameEnum().GetNumEntries()) { LOG(warning, "no docsum format contains field '%s'; copied fields will be empty", inputField); @@ -84,8 +88,12 @@ CopyDFW::insertField(uint32_t /*docid*/, GeneralResult *gres, GetDocsumsState *, int idx = gres->GetClass()->GetIndexFromEnumValue(_inputFieldEnumValue); ResEntry *entry = gres->GetEntry(idx); - if (entry != nullptr && - IsRuntimeCompatible(entry->_type, type)) + if (entry == nullptr) { + auto input_field_value = gres->get_field_value(_input_field_name); + if (input_field_value) { + SummaryFieldConverter::insert_summary_field(false, *input_field_value, target); + } + } else if (IsRuntimeCompatible(entry->_type, type)) { switch (type) { case RES_INT: { @@ -121,7 +129,23 @@ CopyDFW::insertField(uint32_t /*docid*/, GeneralResult *gres, GetDocsumsState *, target.insertLong(valint64); break; } - case RES_JSONSTRING: + case RES_JSONSTRING: { + uint32_t len; + const char *spt; + // resolve field + entry->_resolve_field(&spt, &len); + if (len != 0) { + // note: 'JSONSTRING' really means 'structured data' + vespalib::Slime input_field_as_slime; + size_t d = vespalib::slime::BinaryFormat::decode(vespalib::Memory(spt, len), input_field_as_slime); + if (d != len) { + LOG(warning, "could not decode %u bytes: %zu bytes decoded", len, d); + } + if (d != 0) { + inject(input_field_as_slime.get(), target); + } + } + break; } case RES_FEATUREDATA: case RES_LONG_STRING: case RES_STRING: { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h index ed43dde1c34..bc135404de1 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h @@ -18,13 +18,11 @@ class IDocsumFieldWriter public: using UP = std::unique_ptr<IDocsumFieldWriter>; IDocsumFieldWriter() : _index(0) { } - virtual ~IDocsumFieldWriter() {} + virtual ~IDocsumFieldWriter() = default; - static bool IsBinaryCompatible(ResType a, ResType b) - { return ResultConfig::IsBinaryCompatible(a, b); } - - static bool IsRuntimeCompatible(ResType a, ResType b) - { return ResultConfig::IsRuntimeCompatible(a, b); } + static bool IsRuntimeCompatible(ResType a, ResType b) { + return ResultConfig::IsRuntimeCompatible(a, b); + } virtual bool IsGenerated() const = 0; virtual void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, @@ -72,6 +70,7 @@ class CopyDFW : public IDocsumFieldWriter { private: uint32_t _inputFieldEnumValue; + vespalib::string _input_field_name; public: CopyDFW(); @@ -84,5 +83,4 @@ public: vespalib::slime::Inserter &target) override; }; - } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index e538af3839e..16575a2e9dc 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docsumstate.h" +#include "docsum_field_writer_state.h" #include <vespa/juniper/rpinterface.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/searchcommon/attribute/iattributecontext.h> @@ -11,10 +12,6 @@ #include <vespa/searchlib/parsequery/parse.h> #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <vespa/vespalib/util/issue.h> -#include "docsum_field_writer_state.h" - -#include <vespa/log/log.h> -LOG_SETUP(".searchsummary.docsummary.docsumstate"); using search::common::GeoLocationParser; using search::common::GeoLocationSpec; @@ -24,10 +21,8 @@ namespace search::docsummary { GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) : _args(), - _docsumbuf(nullptr), - _docsumcnt(0), + _docsumbuf(), _kwExtractor(nullptr), - _keywords(nullptr), _callback(callback), _dynteaser(), _attrCtx(), @@ -39,8 +34,7 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) _summaryFeaturesCached(false), _omit_summary_features(false), _rankFeatures(nullptr), - _matching_elements(), - _jsonStringer() + _matching_elements() { _dynteaser._docid = static_cast<uint32_t>(-1); _dynteaser._input = static_cast<uint32_t>(-1); @@ -53,8 +47,6 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) GetDocsumsState::~GetDocsumsState() { - free(_docsumbuf); - free(_keywords); if (_dynteaser._result != nullptr) { juniper::ReleaseResult(_dynteaser._result); } @@ -72,14 +64,6 @@ GetDocsumsState::get_matching_elements(const MatchingElementsFields &matching_el return *_matching_elements; } -vespalib::JSONStringer & -GetDocsumsState::jsonStringer() { - if (!_jsonStringer) { - _jsonStringer = std::make_unique<vespalib::JSONStringer>(); - } - return *_jsonStringer; -} - void GetDocsumsState::parse_locations() { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index c25aca15200..438a8a6d847 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -2,11 +2,9 @@ #pragma once -#include <vespa/searchlib/util/rawbuf.h> -#include <vespa/searchsummary/docsummary/getdocsumargs.h> +#include "getdocsumargs.h" #include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/common/geo_location_spec.h> -#include <vespa/vespalib/util/jsonwriter.h> #include <vespa/vespalib/util/stash.h> namespace juniper { @@ -53,12 +51,8 @@ public: const search::attribute::IAttributeVector * getAttribute(size_t index) const { return _attributes[index]; } GetDocsumArgs _args; // from getdocsums request - - uint32_t *_docsumbuf; // from getdocsums request - uint32_t _docsumcnt; // from getdocsums request - + std::vector<uint32_t> _docsumbuf; // from getdocsums request KeywordExtractor *_kwExtractor; - char *_keywords; // list of keywords from query GetDocsumsStateCallback &_callback; @@ -85,12 +79,12 @@ public: void parse_locations(); // used by SummaryFeaturesDFW - FeatureSet::SP _summaryFeatures; + std::shared_ptr<FeatureSet> _summaryFeatures; bool _summaryFeaturesCached; bool _omit_summary_features; // used by RankFeaturesDFW - FeatureSet::SP _rankFeatures; + std::shared_ptr<FeatureSet> _rankFeatures; // Used by AttributeCombinerDFW and MultiAttrDFW when filtering is enabled std::unique_ptr<search::MatchingElements> _matching_elements; @@ -101,11 +95,7 @@ public: ~GetDocsumsState(); const MatchingElements &get_matching_elements(const MatchingElementsFields &matching_elems_fields); - vespalib::JSONStringer & jsonStringer(); vespalib::Stash& get_stash() noexcept { return _stash; } -private: - // Only used by rank/summary features, so make it lazy - std::unique_ptr<vespalib::JSONStringer> _jsonStringer; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp index be6664e41a3..f17787ff4e5 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp @@ -12,55 +12,32 @@ LOG_SETUP(".searchlib.docsummary.urlresult"); namespace search::docsummary { void -GeneralResult::AllocEntries(uint32_t buflen, bool inplace) +GeneralResult::AllocEntries() { uint32_t cnt = _resClass->GetNumEntries(); - uint32_t needMem = (inplace) - ? cnt * sizeof(ResEntry) - : cnt * sizeof(ResEntry) + buflen + 1; + uint32_t needMem = cnt * sizeof(ResEntry); if (cnt > 0) { _entrycnt = cnt; _entries = (ResEntry *) malloc(needMem); assert(_entries != nullptr); - if (inplace) { - _buf = nullptr; - _bufEnd = nullptr; - } else { - _buf = ((char *)_entries) + cnt * sizeof(ResEntry); - _bufEnd = _buf + buflen + 1; - } memset(_entries, 0, cnt * sizeof(ResEntry)); } else { _entrycnt = 0; _entries = nullptr; - _buf = nullptr; - _bufEnd = nullptr; } } void GeneralResult::FreeEntries() { - uint32_t cnt = _entrycnt; - - // (_buf == nullptr) <=> (_inplace_unpack() || (cnt == 0)) - if (_buf != nullptr) { - for (uint32_t i = 0; i < cnt; i++) { - if (ResultConfig::IsVariableSize(_entries[i]._type) && - !InBuf(_entries[i]._stringval)) - delete [] (_entries[i]._stringval); - } - } - free(_entries); // free '_entries'/'_buf' chunk + free(_entries); // free '_entries' chunk } GeneralResult::GeneralResult(const ResultClass *resClass) : _resClass(resClass), _entrycnt(0), _entries(nullptr), - _buf(nullptr), - _bufEnd(nullptr), _document() { } @@ -117,7 +94,7 @@ GeneralResult::unpack(const char *buf, const size_t buflen) if (_entries != nullptr) FreeEntries(); - AllocEntries(buflen, true); + AllocEntries(); for (uint32_t i = 0; rc && i < _entrycnt; i++) { const ResConfigEntry *entry = _resClass->GetEntry(i); @@ -331,8 +308,6 @@ GeneralResult::unpack(const char *buf, const size_t buflen) FreeEntries(); _entrycnt = 0; _entries = nullptr; - _buf = nullptr; - _bufEnd = nullptr; return false; // FAIL } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.h b/searchsummary/src/vespa/searchsummary/docsummary/general_result.h index b5d07d714ee..a5664a8a606 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.h @@ -21,16 +21,9 @@ private: const ResultClass *_resClass; uint32_t _entrycnt; ResEntry *_entries; - char *_buf; // allocated in same chunk as _entries - char *_bufEnd; // first byte after _buf const document::Document* _document; - bool InBuf(const void *pt) const { - return ((const char *)pt >= _buf && - (const char *)pt < _bufEnd); - } - - void AllocEntries(uint32_t buflen, bool inplace = false); + void AllocEntries(); void FreeEntries(); public: diff --git a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h index 9be90aba672..7c346a4b9bb 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h @@ -4,7 +4,6 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/engine/docsumrequest.h> -#include <vespa/searchlib/engine/propertiesmap.h> namespace search::docsummary { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index 1a2cf24a18d..1fcb0a49be1 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -65,35 +65,6 @@ AbsDistanceDFW::AbsDistanceDFW(const vespalib::string & attrName) : LocationAttrDFW(attrName) { } -double -AbsDistanceDFW::kmMinDistance(uint32_t docid, GetDocsumsState *state, - const std::vector<const GeoLoc *> &locations) -{ - double best = std::numeric_limits<double>::max(); - const auto& attribute = get_attribute(*state); - for (auto location : locations) { - double lat = to_degrees(location->point.y); - double lng = to_degrees(location->point.x); - GeoGcd point{lat, lng}; - int32_t docx = 0; - int32_t docy = 0; - IntegerContent pos; - pos.fill(attribute, docid); - uint32_t numValues = pos.size(); - for (uint32_t i = 0; i < numValues; i++) { - int64_t docxy(pos[i]); - vespalib::geo::ZCurve::decode(docxy, &docx, &docy); - lat = to_degrees(docy); - lng = to_degrees(docx); - double dist = point.km_great_circle_distance(lat, lng); - if (dist < best) { - best = dist; - } - } - } - return best; -} - uint64_t AbsDistanceDFW::findMinDistance(uint32_t docid, GetDocsumsState *state, const std::vector<const GeoLoc *> &locations) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h index 7f6bfe22816..b3e041c1379 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h @@ -36,8 +36,6 @@ public: class AbsDistanceDFW : public LocationAttrDFW { private: - double kmMinDistance(uint32_t docid, GetDocsumsState *state, - const std::vector<const GeoLoc *> &locations); uint64_t findMinDistance(uint32_t docid, GetDocsumsState *state, const std::vector<const GeoLoc *> &locations); public: diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp index e7379296f78..38b58ef94fc 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp @@ -2,28 +2,21 @@ #include "rankfeaturesdfw.h" #include "docsumstate.h" -#include <vespa/searchlib/common/packets.h> #include <vespa/vespalib/data/slime/cursor.h> namespace search::docsummary { -RankFeaturesDFW::RankFeaturesDFW() : - _env(nullptr) +RankFeaturesDFW::RankFeaturesDFW(IDocsumEnvironment * env) : + _env(env) { } RankFeaturesDFW::~RankFeaturesDFW() = default; void -RankFeaturesDFW::init(IDocsumEnvironment * env) -{ - _env = env; -} - -void RankFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, - ResType type, vespalib::slime::Inserter &target) + ResType, vespalib::slime::Inserter &target) { - if (state->_rankFeatures.get() == nullptr) { + if ( !state->_rankFeatures ) { state->_callback.FillRankFeatures(state, _env); if (state->_rankFeatures.get() == nullptr) { // still no rank features to write return; @@ -31,35 +24,16 @@ RankFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, } const FeatureSet::StringVector & names = state->_rankFeatures->getNames(); const FeatureSet::Value * values = state->_rankFeatures->getFeaturesByDocId(docid); - if (type == RES_FEATUREDATA && values != nullptr) { - vespalib::slime::Cursor& obj = target.insertObject(); - for (uint32_t i = 0; i < names.size(); ++i) { - vespalib::Memory name(names[i].c_str(), names[i].size()); - if (values[i].is_data()) { - obj.setData(name, values[i].as_data()); - } else { - obj.setDouble(name, values[i].as_double()); - } - } - return; - } - vespalib::JSONStringer & json(state->jsonStringer()); - if (values != nullptr) { - json.clear(); - json.beginObject(); - for (uint32_t i = 0; i < names.size(); ++i) { - featureDump(json, names[i], values[i].as_double()); - } - json.endObject(); - vespalib::Memory value(json.toString().data(), - json.toString().size()); - if (type == RES_STRING || type == RES_LONG_STRING) { - target.insertString(value); - } - if (type == RES_DATA || type == RES_LONG_DATA) { - target.insertData(value); + if (values == nullptr) { return; } + + vespalib::slime::Cursor& obj = target.insertObject(); + for (uint32_t i = 0; i < names.size(); ++i) { + vespalib::Memory name(names[i].c_str(), names[i].size()); + if (values[i].is_data()) { + obj.setData(name, values[i].as_data()); + } else { + obj.setDouble(name, values[i].as_double()); } - json.clear(); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h index 92d458c7cbb..eab9fab60b2 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h @@ -6,18 +6,16 @@ namespace search::docsummary { -class RankFeaturesDFW : public FeaturesDFW +class RankFeaturesDFW : public ISimpleDFW { private: - RankFeaturesDFW(const RankFeaturesDFW &); - RankFeaturesDFW & operator=(const RankFeaturesDFW &); - IDocsumEnvironment * _env; public: - RankFeaturesDFW(); - ~RankFeaturesDFW(); - void init(IDocsumEnvironment * env); + RankFeaturesDFW(IDocsumEnvironment * env); + RankFeaturesDFW(const RankFeaturesDFW &) = delete; + RankFeaturesDFW & operator=(const RankFeaturesDFW &) = delete; + ~RankFeaturesDFW() override; bool IsGenerated() const override { return true; } void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h index be3bb7570d2..68e5d6848b9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h @@ -71,15 +71,6 @@ struct ResEntry void *_pt; }; - uint32_t _get_length() const { return _len; } - uint32_t _get_real_length() const - { - // precond: IsVariableSize(_type) && _len >= sizeof(uint32_t) - - uint32_t rlen; - memcpy(&rlen, _pt, sizeof(rlen)); - return rlen; - } void _resolve_field(const char **buf, uint32_t *buflen) const { // precond: IsVariableSize(_type) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index 37ecbb6737a..f920b7be0cd 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -2,7 +2,6 @@ #include "summaryfeaturesdfw.h" #include "docsumstate.h" -#include <vespa/searchlib/common/packets.h> #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/log/log.h> @@ -11,24 +10,17 @@ LOG_SETUP(".searchlib.docsummary.summaryfeaturesdfw"); namespace search::docsummary { -SummaryFeaturesDFW::SummaryFeaturesDFW() : - _env(nullptr) +SummaryFeaturesDFW::SummaryFeaturesDFW(IDocsumEnvironment * env) : + _env(env) { } SummaryFeaturesDFW::~SummaryFeaturesDFW() = default; -void -SummaryFeaturesDFW::init(IDocsumEnvironment * env) -{ - _env = env; -} - -static vespalib::string _G_cached("vespa.summaryFeatures.cached"); static vespalib::Memory _M_cached("vespa.summaryFeatures.cached"); void -SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) +SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) { if (state->_omit_summary_features) { return; @@ -41,55 +33,21 @@ SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType } const FeatureSet::StringVector &names = state->_summaryFeatures->getNames(); const FeatureSet::Value *values = state->_summaryFeatures->getFeaturesByDocId(docid); - if (type == RES_FEATUREDATA && values != nullptr) { - vespalib::slime::Cursor& obj = target.insertObject(); - for (uint32_t i = 0; i < names.size(); ++i) { - vespalib::Memory name(names[i].c_str(), names[i].size()); - if (values[i].is_data()) { - obj.setData(name, values[i].as_data()); - } else { - obj.setDouble(name, values[i].as_double()); - } - } - if (state->_summaryFeaturesCached) { - obj.setDouble(_M_cached, 1.0); - } else { - obj.setDouble(_M_cached, 0.0); - } - return; - } - vespalib::JSONStringer & json(state->jsonStringer()); - if (values != nullptr) { - json.clear(); - json.beginObject(); - for (uint32_t i = 0; i < names.size(); ++i) { - featureDump(json, names[i], values[i].as_double()); - } - json.appendKey(_G_cached); - if (state->_summaryFeaturesCached) { - json.appendDouble(1.0); + if (values == nullptr) { return; } + + vespalib::slime::Cursor& obj = target.insertObject(); + for (uint32_t i = 0; i < names.size(); ++i) { + vespalib::Memory name(names[i].c_str(), names[i].size()); + if (values[i].is_data()) { + obj.setData(name, values[i].as_data()); } else { - json.appendDouble(0.0); - } - json.endObject(); - vespalib::Memory value(json.toString().data(), json.toString().size()); - if (type == RES_STRING || type == RES_LONG_STRING) { - target.insertString(value); + obj.setDouble(name, values[i].as_double()); } - if (type == RES_DATA || type == RES_LONG_DATA) { - target.insertData(value); - } - json.clear(); } -} - -void FeaturesDFW::featureDump(vespalib::JSONStringer & json, vespalib::stringref name, double feature) -{ - json.appendKey(name); - if (std::isnan(feature) || std::isinf(feature)) { - json.appendNull(); + if (state->_summaryFeaturesCached) { + obj.setDouble(_M_cached, 1.0); } else { - json.appendDouble(feature); + obj.setDouble(_M_cached, 0.0); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h index 1885ef57399..6c4084b0221 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h @@ -4,30 +4,20 @@ #include "docsumfieldwriter.h" -namespace vespalib { class JSONStringer; } - namespace search::docsummary { class IDocsumEnvironment; -class FeaturesDFW : public ISimpleDFW -{ -protected: - void featureDump(vespalib::JSONStringer & json, vespalib::stringref name, double feature); -}; - -class SummaryFeaturesDFW : public FeaturesDFW +class SummaryFeaturesDFW : public ISimpleDFW { private: - SummaryFeaturesDFW(const SummaryFeaturesDFW &); - SummaryFeaturesDFW & operator=(const SummaryFeaturesDFW &); - IDocsumEnvironment * _env; public: - SummaryFeaturesDFW(); + SummaryFeaturesDFW(IDocsumEnvironment * env); + SummaryFeaturesDFW(const SummaryFeaturesDFW &) = delete; + SummaryFeaturesDFW & operator=(const SummaryFeaturesDFW &) = delete; ~SummaryFeaturesDFW() override; - void init(IDocsumEnvironment * env); bool IsGenerated() const override { return true; } void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp index c107e5c8739..6082c82e863 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp @@ -593,5 +593,11 @@ SummaryFieldConverter::convert_field_with_filter(bool markup, return SummaryFieldValueConverter(markup, sub_conv).convert(value); } +void +SummaryFieldConverter::insert_summary_field(bool markup, const FieldValue& value, vespalib::slime::Inserter& inserter) +{ + SlimeFiller visitor(inserter, markup); + value.accept(visitor); +} } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h index 526b8c64130..4367dbcd109 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h @@ -4,6 +4,8 @@ #include <vespa/document/fieldvalue/fieldvalue.h> +namespace vespalib::slime { struct Inserter; } + namespace search::docsummary { /** @@ -22,6 +24,8 @@ public: static document::FieldValue::UP convert_field_with_filter(bool markup, const document::FieldValue& value, const std::vector<uint32_t>& matching_elems); + + static void insert_summary_field(bool markup, const document::FieldValue& value, vespalib::slime::Inserter& inserter); }; } diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index 86cad2ad012..b73212b96f6 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -99,6 +99,8 @@ config-model-api-jar-with-dependencies.jar, config-model-jar-with-dependencies.jar, container-disc-jar-with-dependencies.jar, + container-search-and-docproc-jar-with-dependencies.jar, + linguistics-components-jar-with-dependencies.jar, model-evaluation-jar-with-dependencies.jar, model-integration-jar-with-dependencies.jar, </discPreInstallBundle> diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def index 65a3cd718ec..c22cf7b8b01 100644 --- a/storage/src/vespa/storage/config/stor-communicationmanager.def +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -51,7 +51,7 @@ mbus.dispatch_on_encode bool default=true restart ## Enable to use above thread pool for decoding replies ## False will use network(fnet) thread ## Todo: Change default once verified in large scale deployment. -mbus.dispatch_on_decode bool default=false restart +mbus.dispatch_on_decode bool default=true restart ## Skip messenger thread on reply ## Experimental @@ -65,11 +65,6 @@ mbus.skip_request_thread bool default=false restart ## Experimental skip_thread bool default=false -## Whether to use direct P2P RPC protocol for all StorageAPI communication -## instead of going via MessageBus. -## Deprecated, and will soon be gone as it is default on. -use_direct_storageapi_rpc bool default=true - ## The number of network (FNET) threads used by the shared rpc resource. rpc.num_network_threads int default=2 restart diff --git a/storage/src/vespa/storage/distributor/update_metric_set.cpp b/storage/src/vespa/storage/distributor/update_metric_set.cpp index 70374cab6d2..82f55d3e819 100644 --- a/storage/src/vespa/storage/distributor/update_metric_set.cpp +++ b/storage/src/vespa/storage/distributor/update_metric_set.cpp @@ -7,7 +7,7 @@ namespace storage { using metrics::MetricSet; UpdateMetricSet::UpdateMetricSet(MetricSet* owner) - : PersistenceOperationMetricSet("updates.sum", owner), + : PersistenceOperationMetricSet("updates", owner), diverging_timestamp_updates("diverging_timestamp_updates", {}, "Number of updates that report they were performed against " "divergent version timestamps on different replicas", this), diff --git a/storage/src/vespa/storage/distributor/visitormetricsset.cpp b/storage/src/vespa/storage/distributor/visitormetricsset.cpp index c449b82e93c..c94dc025fa1 100644 --- a/storage/src/vespa/storage/distributor/visitormetricsset.cpp +++ b/storage/src/vespa/storage/distributor/visitormetricsset.cpp @@ -7,7 +7,7 @@ namespace storage { using metrics::MetricSet; VisitorMetricSet::VisitorMetricSet(MetricSet* owner) - : PersistenceOperationMetricSet("visitor.sum", owner), + : PersistenceOperationMetricSet("visitor", owner), buckets_per_visitor("buckets_per_visitor", {}, "The number of sub buckets visited as part of a " "single client visitor command", this), diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 975e9361072..5a3ebebcd9e 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -49,7 +49,7 @@ void CommunicationManager::receiveStorageReply(const std::shared_ptr<api::StorageReply>& reply) { assert(reply); - enqueue_or_process(reply); + process(reply); } namespace { @@ -104,7 +104,7 @@ CommunicationManager::handleMessage(std::unique_ptr<mbus::Message> msg) cmd->setTrace(docMsgPtr->steal_trace()); cmd->setTransportContext(std::make_unique<StorageTransportContext>(std::move(docMsgPtr))); - enqueue_or_process(std::move(cmd)); + process(std::move(cmd)); } else if (protocolName == mbusprot::StorageProtocol::NAME) { std::unique_ptr<mbusprot::StorageCommand> storMsgPtr(static_cast<mbusprot::StorageCommand*>(msg.release())); @@ -116,7 +116,7 @@ CommunicationManager::handleMessage(std::unique_ptr<mbus::Message> msg) cmd->setTrace(storMsgPtr->steal_trace()); cmd->setTransportContext(std::make_unique<StorageTransportContext>(std::move(storMsgPtr))); - enqueue_or_process(std::move(cmd)); + process(std::move(cmd)); } else { LOGBM(warning, "Received unsupported message type %d for protocol '%s'", msg->getType(), msg->getProtocol().c_str()); @@ -268,8 +268,7 @@ CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, co _configUri(configUri), _closed(false), _docApiConverter(configUri, std::make_shared<PlaceHolderBucketResolver>()), - _thread(), - _skip_thread(false) + _thread() { _component.registerMetricUpdateHook(*this, framework::SecondTime(5)); _component.registerMetric(_metrics); @@ -372,7 +371,6 @@ CommunicationManager::configureMessageBusLimits(const CommunicationManagerConfig void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> config) { // Only allow dynamic (live) reconfiguration of message bus limits. - _skip_thread = config->skipThread; if (_mbus) { configureMessageBusLimits(*config); if (_mbus->getRPCNetwork().getPort() != config->mbusport) { @@ -411,8 +409,6 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> CommunicationManagerConfig::Mbus::Compress::getTypeName(config->mbus.compress.type).c_str()); params.setCompressionConfig(CompressionConfig(compressionType, config->mbus.compress.level, 90, config->mbus.compress.limit)); - params.setSkipRequestThread(config->mbus.skipRequestThread); - params.setSkipReplyThread(config->mbus.skipReplyThread); // Configure messagebus here as we for legacy reasons have // config here. @@ -472,18 +468,6 @@ CommunicationManager::process(const std::shared_ptr<api::StorageMessage>& msg) } } -void -CommunicationManager::enqueue_or_process(std::shared_ptr<api::StorageMessage> msg) -{ - assert(msg); - if (_skip_thread.load(std::memory_order_relaxed)) { - LOG(spam, "Process storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); - process(msg); - } else { - dispatch_async(std::move(msg)); - } -} - void CommunicationManager::dispatch_sync(std::shared_ptr<api::StorageMessage> msg) { LOG(spam, "Direct dispatch of storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); process(std::move(msg)); diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index 31c6fa00f0e..d52fb56cf20 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -121,10 +121,8 @@ private: std::atomic<bool> _closed; DocumentApiConverter _docApiConverter; framework::Thread::UP _thread; - std::atomic<bool> _skip_thread; void updateMetrics(const MetricLockGuard &) override; - void enqueue_or_process(std::shared_ptr<api::StorageMessage> msg); // Test needs access to configure() for live reconfig testing. friend struct CommunicationManagerTest; diff --git a/storage/src/vespa/storageapi/mbusprot/storageprotocol.h b/storage/src/vespa/storageapi/mbusprot/storageprotocol.h index 3f36ac42117..591009c9832 100644 --- a/storage/src/vespa/storageapi/mbusprot/storageprotocol.h +++ b/storage/src/vespa/storageapi/mbusprot/storageprotocol.h @@ -22,7 +22,6 @@ public: mbus::IRoutingPolicy::UP createPolicy(const mbus::string& name, const mbus::string& param) const override; mbus::Blob encode(const vespalib::Version&, const mbus::Routable&) const override; mbus::Routable::UP decode(const vespalib::Version&, mbus::BlobRef) const override; - bool requireSequencing() const override { return true; } private: ProtocolSerialization5_0 _serializer5_0; ProtocolSerialization5_1 _serializer5_1; diff --git a/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp b/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp index c8e2bef1016..9715b44b27b 100644 --- a/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp +++ b/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp @@ -283,11 +283,13 @@ HitCollectorTest::testFeatureSet() MyRankProgram rankProgram; FeatureResolver resolver(rankProgram.get_resolver()); - search::FeatureSet::SP sf = hc.getFeatureSet(rankProgram, resolver); + search::StringStringMap renames; + renames["bar"] = "qux"; + search::FeatureSet::SP sf = hc.getFeatureSet(rankProgram, resolver, renames); EXPECT_EQUAL(sf->getNames().size(), 3u); EXPECT_EQUAL(sf->getNames()[0], "foo"); - EXPECT_EQUAL(sf->getNames()[1], "bar"); + EXPECT_EQUAL(sf->getNames()[1], "qux"); EXPECT_EQUAL(sf->getNames()[2], "baz"); EXPECT_EQUAL(sf->numFeatures(), 3u); EXPECT_EQUAL(sf->numDocs(), 3u); diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp index d11659cfb77..10e6c6aa68a 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp @@ -143,7 +143,8 @@ HitCollector::fillSearchResult(vdslib::SearchResult & searchResult) FeatureSet::SP HitCollector::getFeatureSet(IRankProgram &rankProgram, - const search::fef::FeatureResolver &resolver) + const search::fef::FeatureResolver &resolver, + const search::StringStringMap &feature_rename_map) { if (resolver.num_features() == 0 || _hits.empty()) { return FeatureSet::SP(new FeatureSet()); @@ -152,7 +153,12 @@ HitCollector::getFeatureSet(IRankProgram &rankProgram, std::vector<vespalib::string> names; names.reserve(resolver.num_features()); for (size_t i = 0; i < resolver.num_features(); ++i) { - names.emplace_back(resolver.name_of(i)); + vespalib::string name = resolver.name_of(i); + auto iter = feature_rename_map.find(name); + if (iter != feature_rename_map.end()) { + name = iter->second; + } + names.emplace_back(name); } FeatureSet::SP retval = FeatureSet::SP(new FeatureSet(names, _hits.size())); for (const Hit & hit : _hits) { diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.h b/streamingvisitors/src/vespa/searchvisitor/hitcollector.h index a7bfa5a51b1..d6a05dc4f9e 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.h +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/searchlib/common/featureset.h> +#include <vespa/searchlib/common/stringmap.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/vdslib/container/searchresult.h> #include <vespa/vsm/common/docsum.h> @@ -132,7 +133,8 @@ public: * @param resolver feature resolver, gives feature names and values **/ search::FeatureSet::SP getFeatureSet(IRankProgram &rankProgram, - const search::fef::FeatureResolver &resolver); + const search::fef::FeatureResolver &resolver, + const search::StringStringMap &feature_rename_map); }; diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp index e31341c466c..24925bd67ee 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp @@ -212,7 +212,7 @@ RankProcessor::calculateFeatureSet() search::fef::FeatureResolver resolver(rankProgram.get_seeds(false)); LOG(debug, "Feature handles: numNames(%ld)", resolver.num_features()); RankProgramWrapper wrapper(*_match_data); - FeatureSet::SP sf = _hitCollector->getFeatureSet(wrapper, resolver); + FeatureSet::SP sf = _hitCollector->getFeatureSet(wrapper, resolver, _rankSetup.get_feature_rename_map()); LOG(debug, "Feature set: numFeatures(%u), numDocs(%u)", sf->numFeatures(), sf->numDocs()); return sf; } diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index 4e050affec6..dee620738c6 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -369,7 +369,7 @@ TEST(DistributionTest, testHighSplitBit) for (uint32_t bits = 33; bits < 36; ++bits) { uint64_t base = 0x23456789; - base |= (1 << bits); + base |= (1L << bits); document::BucketId bid1 = document::BucketId(bits, base); document::BucketId bid2 = document::BucketId(bits, base); diff --git a/vdslib/src/vespa/vdslib/container/searchresult.cpp b/vdslib/src/vespa/vdslib/container/searchresult.cpp index 12d5544d6ef..6989fbca8e0 100644 --- a/vdslib/src/vespa/vdslib/container/searchresult.cpp +++ b/vdslib/src/vespa/vdslib/container/searchresult.cpp @@ -63,7 +63,9 @@ size_t BlobContainer::append(const void * v, size_t sz) if (getSize() > _blob.size()) { _blob.realloc(getSize()*2); } - memcpy(_blob.str() + _offsets[index], v, sz); + if (sz > 0) { + memcpy(_blob.str() + _offsets[index], v, sz); + } return index; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index a4045016b78..7f16505c500 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -103,11 +103,11 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, Set<RoleAction> roleActions, OAuthCredentials oAuthCredentials) { URI uri = zmsUrl.resolve(String.format("domain/%s/provDomain/%s/provService/%s/resourceGroup/%s", tenantDomain.getName(), providerService.getDomainName(), providerService.getName(), resourceGroup)); - HttpUriRequest request = RequestBuilder.put() + RequestBuilder builder = RequestBuilder.put() .setUri(uri) - .addHeader(createCookieHeader(oAuthCredentials)) - .setEntity(toJsonStringEntity(new ResourceGroupRolesEntity(providerService, tenantDomain, roleActions, resourceGroup))) - .build(); + .setEntity(toJsonStringEntity(new ResourceGroupRolesEntity(providerService, tenantDomain, roleActions, resourceGroup))); + if (oAuthCredentials != null) builder.addHeader(createCookieHeader(oAuthCredentials)); + HttpUriRequest request = builder.build(); execute(request, response -> readEntity(response, Void.class)); // Note: The ZMS API will actually return a json object that is similar to ProviderResourceGroupRolesRequestEntity } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index 13a61d65d78..197af753442 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -116,22 +116,23 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public ZToken getRoleToken(AthenzDomain domain) { - return getRoleToken(domain, null); + public ZToken getRoleToken(AthenzDomain domain, Duration expiry) { + return getRoleToken(domain, null, expiry); } @Override - public ZToken getRoleToken(AthenzRole athenzRole) { - return getRoleToken(athenzRole.domain(), athenzRole.roleName()); + public ZToken getRoleToken(AthenzRole athenzRole, Duration expiry) { + return getRoleToken(athenzRole.domain(), athenzRole.roleName(), expiry); } - private ZToken getRoleToken(AthenzDomain domain, String roleName) { + private ZToken getRoleToken(AthenzDomain domain, String roleName, Duration expiry) { URI uri = ztsUrl.resolve(String.format("domain/%s/token", domain.getName())); RequestBuilder requestBuilder = RequestBuilder.get(uri) .addHeader("Content-Type", "application/json"); if (roleName != null) { requestBuilder.addParameter("role", roleName); } + requestBuilder.addParameter("maxExpiryTime", Long.toString(expiry.getSeconds())); HttpUriRequest request = requestBuilder.build(); return execute(request, response -> { RoleTokenResponseEntity roleTokenResponseEntity = readEntity(response, RoleTokenResponseEntity.class); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java index e52abc4193b..30c8ab2fd50 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java @@ -68,15 +68,37 @@ public interface ZtsClient extends AutoCloseable { * @param domain Target domain * @return A role token */ - ZToken getRoleToken(AthenzDomain domain); + default ZToken getRoleToken(AthenzDomain domain) { + return getRoleToken(domain, Duration.ofHours(1)); + } + + /** + * Fetch a role token for the target domain + * + * @param domain Target domain + * @param tokenExpiry Token expiry + * @return A role token + */ + ZToken getRoleToken(AthenzDomain domain, Duration tokenExpiry); + + /** + * Fetch a role token for the target role + * + * @param athenzRole Target role + * @return A role token + */ + default ZToken getRoleToken(AthenzRole athenzRole) { + return getRoleToken(athenzRole, Duration.ofHours(1)); + } /** * Fetch a role token for the target role * * @param athenzRole Target role + * @param tokenExpiry Token expiry * @return A role token */ - ZToken getRoleToken(AthenzRole athenzRole); + ZToken getRoleToken(AthenzRole athenzRole, Duration tokenExpiry); /** * Fetch an access token for the target domain diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java index 9d6c9f5e8d5..21650d72d6f 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java @@ -41,17 +41,4 @@ public interface ServiceIdentityProvider { */ Path privateKeyPath(); - /** - * @return Path to Athenz truststore in PEM format - */ - Path athenzTruststorePath(); - - /** - * The client truststore contains the Athenz certificates from {@link #athenzTruststorePath()} - * and additional certificate authorities that issues trusted server certificates. - * - * @return Path to client truststore in PEM format - */ - Path clientTruststorePath(); - } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index 3c1a59dab51..e76384d4d8b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -27,41 +27,33 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde private final AthenzIdentity service; private final Path certificateFile; private final Path privateKeyFile; - private final Path clientTruststoreFile; - private final Path athenzTruststoreFile; @Inject public SiaIdentityProvider(SiaProviderConfig config) { this(new AthenzService(config.athenzDomain(), config.athenzService()), SiaUtils.getPrivateKeyFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())), SiaUtils.getCertificateFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())), - Paths.get(config.athenzTruststorePath()), Paths.get(config.trustStorePath())); } public SiaIdentityProvider(AthenzIdentity service, Path siaPath, - Path athenzTruststoreFile, Path clientTruststoreFile) { this(service, SiaUtils.getPrivateKeyFile(siaPath, service), SiaUtils.getCertificateFile(siaPath, service), - athenzTruststoreFile, clientTruststoreFile); } public SiaIdentityProvider(AthenzIdentity service, Path privateKeyFile, Path certificateFile, - Path athenzTruststoreFile, Path clientTruststoreFile) { this.service = service; this.keyManager = AutoReloadingX509KeyManager.fromPemFiles(privateKeyFile, certificateFile); this.sslContext = createIdentitySslContext(keyManager, clientTruststoreFile); this.certificateFile = certificateFile; this.privateKeyFile = privateKeyFile; - this.athenzTruststoreFile = athenzTruststoreFile; - this.clientTruststoreFile = clientTruststoreFile; } @Override @@ -77,8 +69,6 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde @Override public X509CertificateWithKey getIdentityCertificateWithKey() { return keyManager.getCurrentCertificateWithKey(); } @Override public Path certificatePath() { return certificateFile; } @Override public Path privateKeyPath() { return privateKeyFile; } - @Override public Path athenzTruststorePath() { return athenzTruststoreFile; } - @Override public Path clientTruststorePath() { return clientTruststoreFile; } private static SSLContext createIdentitySslContext(AutoReloadingX509KeyManager keyManager, Path trustStoreFile) { return new SslContextBuilder() diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index 418f7ec024b..c92f7259e77 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -68,7 +68,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen static final Duration UPDATE_PERIOD = Duration.ofDays(1); static final Duration AWAIT_TERMINTATION_TIMEOUT = Duration.ofSeconds(90); private final static Duration ROLE_SSL_CONTEXT_EXPIRY = Duration.ofHours(2); - private final static Duration ROLE_TOKEN_EXPIRY = Duration.ofMinutes(30); + // TODO CMS expects 10min or less token ttl. Use 10min default until we have configurable expiry + private final static Duration ROLE_TOKEN_EXPIRY = Duration.ofMinutes(10); // TODO Make path to trust store paths config private static final Path CLIENT_TRUST_STORE = Paths.get("/opt/yahoo/share/ssl/certs/yahoo_certificate_bundle.pem"); @@ -204,16 +205,12 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen @Override public Path privateKeyPath() { return athenzCredentialsService.privateKeyPath(); } - @Override public Path athenzTruststorePath() { return ATHENZ_TRUST_STORE; } - - @Override public Path clientTruststorePath() { return CLIENT_TRUST_STORE; } - @Override public SSLContext getRoleSslContext(String domain, String role) { try { AthenzRole athenzRole = new AthenzRole(new AthenzDomain(domain), role); // Make sure to request a certificate which triggers creating a new key manager for this role - X509Certificate x509Certificate = roleSslCertCache.get(athenzRole); + X509Certificate x509Certificate = getRoleCertificate(athenzRole); MutableX509KeyManager keyManager = roleKeyManagerCache.get(athenzRole); return new SslContextBuilder() .withKeyManager(keyManager) @@ -278,6 +275,19 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen return Collections.singletonList(credentials.getCertificate()); } + @Override + public X509Certificate getRoleCertificate(String domain, String role) { + return getRoleCertificate(new AthenzRole(new AthenzDomain(domain), role)); + } + + private X509Certificate getRoleCertificate(AthenzRole athenzRole) { + try { + return roleSslCertCache.get(athenzRole); + } catch (Exception e) { + throw new AthenzIdentityProviderException("Could not retrieve role certificate: " + e.getMessage(), e); + } + } + private void updateIdentityCredentials(AthenzCredentials credentials) { this.credentials = credentials; this.identityKeyManager.updateKeystore( @@ -308,13 +318,13 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private ZToken createRoleToken(AthenzRole athenzRole) { try (ZtsClient client = createZtsClient()) { - return client.getRoleToken(athenzRole); + return client.getRoleToken(athenzRole, ROLE_TOKEN_EXPIRY); } } private ZToken createRoleToken(AthenzDomain domain) { try (ZtsClient client = createZtsClient()) { - return client.getRoleToken(domain); + return client.getRoleToken(domain, ROLE_TOKEN_EXPIRY); } } @@ -346,7 +356,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private static SiaIdentityProvider createNodeIdentityProvider(IdentityConfig config) { return new SiaIdentityProvider( - new AthenzService(config.nodeIdentityName()), SiaUtils.DEFAULT_SIA_DIRECTORY, ATHENZ_TRUST_STORE, CLIENT_TRUST_STORE); + new AthenzService(config.nodeIdentityName()), SiaUtils.DEFAULT_SIA_DIRECTORY, CLIENT_TRUST_STORE); } private boolean isExpired(AthenzCredentials credentials) { diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java index 4e3c81c0f39..f502951572c 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java @@ -50,7 +50,6 @@ public class SiaIdentityProviderTest { new AthenzService("domain", "service-name"), keyFile.toPath(), certificateFile.toPath(), - trustStoreFile.toPath(), trustStoreFile.toPath()); assertNotNull(provider.getIdentitySslContext()); @@ -74,7 +73,6 @@ public class SiaIdentityProviderTest { new AthenzService("domain", "service-name"), keyFile.toPath(), certificateFile.toPath(), - trustStoreFile.toPath(), trustStoreFile.toPath()); assertNotNull(provider.getIdentitySslContext()); diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/BenchmarkingCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/BenchmarkingCluster.java index f59eee25d42..0a1ad1ee9b7 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/BenchmarkingCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/BenchmarkingCluster.java @@ -4,6 +4,7 @@ package ai.vespa.feed.client.impl; import ai.vespa.feed.client.HttpResponse; import ai.vespa.feed.client.OperationStats; +import java.time.Instant; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -80,12 +81,13 @@ public class BenchmarkingCluster implements Cluster { } private OperationStats getStats() { + long requests = this.requests.get(); + Map<Integer, Long> responses = new HashMap<>(); for (int code = 0; code < responsesByCode.length; code++) if (responsesByCode[code] > 0) responses.put(code, responsesByCode[code]); - long requests = this.requests.get(); return new OperationStats(requests, responses, exceptions, @@ -100,6 +102,14 @@ public class BenchmarkingCluster implements Cluster { @Override public void close() { delegate.close(); + Instant doom = Instant.now().plusSeconds(10); + while (Instant.now().isBefore(doom) && getStats().inflight() != 0) + try { + Thread.sleep(10); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } executor.shutdown(); } diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java index 4bf40323193..b1ca6c84b75 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java @@ -197,7 +197,7 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { nodeObject.setString("name", node.name()); nodeObject.setString("status", node.status().name()); nodeObject.setLong("start", node.start().toEpochMilli()); - nodeObject.setLong("end", node.duration().toMillis()); + nodeObject.setLong("duration", node.duration().toMillis()); } static void toSlime(Cursor nodeObject, TestNode node) { @@ -205,7 +205,7 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { nodeObject.setString("name", node.name()); nodeObject.setString("status", node.status().name()); nodeObject.setLong("start", node.start().toEpochMilli()); - nodeObject.setLong("end", node.duration().toMillis()); + nodeObject.setLong("duration", node.duration().toMillis()); } static void toSlime(Cursor nodeObject, OutputNode node) { diff --git a/vespa-osgi-testrunner/src/test/resources/report.json b/vespa-osgi-testrunner/src/test/resources/report.json index 69b11b40ed9..9c41a83a6b5 100644 --- a/vespa-osgi-testrunner/src/test/resources/report.json +++ b/vespa-osgi-testrunner/src/test/resources/report.json @@ -4,28 +4,28 @@ "name": "Production test", "status": "error", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "container", "name": "JUnit Jupiter", "status": "error", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "container", "name": "SampleTest", "status": "error", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "test", "name": "aborted()", "status": "aborted", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "output", @@ -44,7 +44,7 @@ "name": "error()", "status": "error", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "output", @@ -68,7 +68,7 @@ "name": "failing()", "status": "failed", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "output", @@ -97,14 +97,14 @@ "name": "ignored()", "status": "skipped", "start": 0, - "end": 0 + "duration": 0 }, { "type": "test", "name": "inconclusive(TestReporter)", "status": "inconclusive", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "output", @@ -133,7 +133,7 @@ "name": "successful()", "status": "successful", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "output", @@ -168,14 +168,14 @@ "name": "Inner", "status": "failed", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "container", "name": "others()", "status": "failed", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "output", @@ -192,7 +192,7 @@ "name": "second", "status": "successful", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "output", @@ -211,7 +211,7 @@ "name": "third", "status": "failed", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "failure", @@ -227,7 +227,7 @@ "name": "first()", "status": "successful", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "output", @@ -248,14 +248,14 @@ "name": "Skipped", "status": "skipped", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "test", "name": "disabled()", "status": "skipped", "start": 0, - "end": 0 + "duration": 0 } ] } @@ -268,21 +268,21 @@ "name": "JUnit Jupiter", "status": "error", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "container", "name": "FailingTestAndBothAftersTest", "status": "error", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "test", "name": "test()", "status": "failed", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "failure", @@ -303,7 +303,7 @@ "name": "WrongBeforeAllTest", "status": "error", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "failure", @@ -315,14 +315,14 @@ "name": "fest()", "status": "skipped", "start": 0, - "end": 0 + "duration": 0 }, { "type": "test", "name": "test()", "status": "skipped", "start": 0, - "end": 0 + "duration": 0 } ] }, @@ -331,14 +331,14 @@ "name": "FailingExtensionTest", "status": "error", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "test", "name": "test()", "status": "error", "start": 0, - "end": 0, + "duration": 0, "children": [ { "type": "failure", diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt index c80f22fdb38..613215c46d3 100644 --- a/vespabase/CMakeLists.txt +++ b/vespabase/CMakeLists.txt @@ -35,6 +35,25 @@ install(FILES conf/java.security.override DESTINATION conf/vespa) configure_file(conf/default-env.txt.in conf/default-env.txt @ONLY) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/conf/default-env.txt DESTINATION conf/vespa) -vespa_install_empty_tmp_dir(logs/vespa) -vespa_install_empty_tmp_dir(tmp/vespa) +install(DIRECTORY DESTINATION logs/vespa) +install(DIRECTORY DESTINATION logs/vespa/access) +install(DIRECTORY DESTINATION tmp/vespa) +install(DIRECTORY DESTINATION var/crash) install(DIRECTORY DESTINATION var/db/vespa) +install(DIRECTORY DESTINATION var/db/vespa/config_server) +install(DIRECTORY DESTINATION var/db/vespa/config_server/serverdb) +install(DIRECTORY DESTINATION var/db/vespa/config_server/serverdb/tenants) +install(DIRECTORY DESTINATION var/db/vespa/filedistribution) +install(DIRECTORY DESTINATION var/db/vespa/index) +install(DIRECTORY DESTINATION var/db/vespa/search) +install(DIRECTORY DESTINATION var/db/vespa/tmp) +install(DIRECTORY DESTINATION var/jdisc_container) +install(DIRECTORY DESTINATION var/run) +install(DIRECTORY DESTINATION var/vespa) +install(DIRECTORY DESTINATION var/vespa/application) +install(DIRECTORY DESTINATION var/vespa/bundlecache) +install(DIRECTORY DESTINATION var/vespa/bundlecache/configserver) +install(DIRECTORY DESTINATION var/vespa/cache) +install(DIRECTORY DESTINATION var/vespa/cache/config) +install(DIRECTORY DESTINATION var/zookeeper) +install(DIRECTORY DESTINATION var/zookeeper/version-2) diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index a4d0ad415fa..ff28b31ca2b 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -77,6 +77,10 @@ findhost [ "$VESPA_HOME" ] || { echo "Missing VESPA_HOME variable" 1>&2; exit 1; } if [ "$VESPA_USER" = "" ]; then VESPA_USER=$(id -run) +else + if [ "$VESPA_GROUP" = "" ]; then + VESPA_GROUP=$(id -gn $VESPA_USER) + fi fi if [ "$VESPA_GROUP" = "" ]; then VESPA_GROUP=$(id -rgn) @@ -101,8 +105,8 @@ fixdir () { fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa/configserver fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa/access +fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa/configserver fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa/search fixdir ${VESPA_USER} ${VESPA_GROUP} 755 tmp fixdir ${VESPA_USER} ${VESPA_GROUP} 755 tmp/vespa @@ -123,7 +127,7 @@ fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/vespa fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/vespa/application fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/vespa/bundlecache fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/vespa/bundlecache/configserver -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/vespa/cache/config/ +fixdir ${VESPA_USER} ${VESPA_GROUP} 755 var/vespa/cache/config if [ "${VESPA_UNPRIVILEGED}" != yes ]; then chown -hR ${VESPA_USER} logs/vespa diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp index b0c5545f3ac..f70ffcc2655 100644 --- a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp +++ b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp @@ -10,7 +10,6 @@ #include <vespa/messagebus/routing/routingtable.h> #include <vespa/messagebus/routing/routedirective.h> #include <vespa/messagebus/rpcmessagebus.h> -#include <vespa/messagebus/network/rpcsendv1.h> #include <vespa/messagebus/network/rpcsendv2.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/config/common/exceptions.h> @@ -19,8 +18,10 @@ #include <vespa/fnet/frt/supervisor.h> using config::ConfigGetter; +using config::InvalidConfigException; using messagebus::MessagebusConfig; using document::DocumentTypeRepo; +using namespace vespalib::make_string_short; namespace vesparoute { @@ -60,13 +61,11 @@ Application::main(int argc, char **argv) // _P_A_R_A_N_O_I_A_ mbus::RoutingTable::SP table = _mbus->getRoutingTable(_params.getProtocol()); if ( ! table) { - throw config::InvalidConfigException(vespalib::make_string("There is no routing table for protocol '%s'.", - _params.getProtocol().c_str())); + throw InvalidConfigException(fmt("There is no routing table for protocol '%s'.", _params.getProtocol().c_str())); } for (const std::string & hop : _params.getHops()) { if (table->getHop(hop) == NULL) { - throw config::InvalidConfigException(vespalib::make_string("There is no hop named '%s' for protocol '%s'.", - hop.c_str(), _params.getProtocol().c_str())); + throw InvalidConfigException(fmt("There is no hop named '%s' for protocol '%s'.", hop.c_str(), _params.getProtocol().c_str())); } } @@ -109,7 +108,7 @@ Application::parseArgs(int argc, char **argv) if (++arg < argc) { _params.setDocumentTypesConfigId(argv[arg]); } else { - throw config::InvalidConfigException("Missing value for parameter 'documenttypesconfigid'."); + throw InvalidConfigException("Missing value for parameter 'documenttypesconfigid'."); } } else if (strcasecmp(argv[arg], "--dump") == 0) { _params.setDump(true); @@ -121,7 +120,7 @@ Application::parseArgs(int argc, char **argv) if (++arg < argc) { _params.getHops().push_back(argv[arg]); } else { - throw config::InvalidConfigException("Missing value for parameter 'hop'."); + throw InvalidConfigException("Missing value for parameter 'hop'."); } } else if (strcasecmp(argv[arg], "--hops") == 0) { _params.setListHops(true); @@ -129,25 +128,25 @@ Application::parseArgs(int argc, char **argv) if (++arg < argc) { _params.getRPCNetworkParams().setIdentity(mbus::Identity(argv[arg])); } else { - throw config::InvalidConfigException("Missing value for parameter 'identity'."); + throw InvalidConfigException("Missing value for parameter 'identity'."); } } else if (strcasecmp(argv[arg], "--listenport") == 0) { if (++arg < argc) { _params.getRPCNetworkParams().setListenPort(atoi(argv[arg])); } else { - throw config::InvalidConfigException("Missing value for parameter 'listenport'."); + throw InvalidConfigException("Missing value for parameter 'listenport'."); } } else if (strcasecmp(argv[arg], "--protocol") == 0) { if (++arg < argc) { _params.setProtocol(argv[arg]); } else { - throw config::InvalidConfigException("Missing value for parameter 'protocol'."); + throw InvalidConfigException("Missing value for parameter 'protocol'."); } } else if (strcasecmp(argv[arg], "--route") == 0) { if (++arg < argc) { _params.getRoutes().push_back(argv[arg]); } else { - throw config::InvalidConfigException("Missing value for parameter 'route'."); + throw InvalidConfigException("Missing value for parameter 'route'."); } } else if (strcasecmp(argv[arg], "--routes") == 0) { _params.setListRoutes(true); @@ -155,7 +154,7 @@ Application::parseArgs(int argc, char **argv) if (++arg < argc) { _params.setRoutingConfigId(argv[arg]); } else { - throw config::InvalidConfigException("Missing value for parameter 'routingconfigid'."); + throw InvalidConfigException("Missing value for parameter 'routingconfigid'."); } } else if (strcasecmp(argv[arg], "--services") == 0) { _params.setListServices(true); @@ -163,12 +162,12 @@ Application::parseArgs(int argc, char **argv) if (++arg < argc) { _params.setSlobrokId(argv[arg]); } else { - throw config::InvalidConfigException("Missing value for parameter 'slobrokconfigid'."); + throw InvalidConfigException("Missing value for parameter 'slobrokconfigid'."); } } else if (strcasecmp(argv[arg], "--verify") == 0) { _params.setVerify(true); } else { - throw config::InvalidConfigException(vespalib::make_string("Unknown option '%s'.", argv[arg])); + throw InvalidConfigException(fmt("Unknown option '%s'.", argv[arg])); } } return true; @@ -206,7 +205,7 @@ Application::verifyRoute(const mbus::Route &route, std::set<std::string> &errors for (std::set<std::string>::iterator err = hopErrors.begin(); err != hopErrors.end(); ++err) { - errors.insert(vespalib::make_string("for hop '%s', %s", str.c_str(), err->c_str())); + errors.insert(fmt("for hop '%s', %s", str.c_str(), err->c_str())); } } } @@ -237,7 +236,7 @@ Application::verifyHop(const mbus::HopBlueprint &hop, std::set<std::string> &err if (hop.getDirective(0)->getType() == mbus::IHopDirective::TYPE_ROUTE) { const mbus::RouteDirective &dir = static_cast<const mbus::RouteDirective &>(*hop.getDirective(0)); if (table.getRoute(dir.getName()) == nullptr) { - errors.insert(vespalib::make_string("route '%s' not found", dir.getName().c_str())); + errors.insert(fmt("route '%s' not found", dir.getName().c_str())); return false; } else { return true; @@ -495,8 +494,7 @@ Application::isService(FRT_Supervisor &frt, const std::string &spec) const FRT_StringValue *retList = req->GetReturn()->GetValue(2)._string_array._pt; for (uint32_t i = 0; i < numMethods; ++i) { - if (mbus::RPCSendV1::isCompatible(methods[i]._str,argList[i]._str, retList[i]._str) || - mbus::RPCSendV2::isCompatible(methods[i]._str,argList[i]._str, retList[i]._str)) { + if (mbus::RPCSendV2::isCompatible(methods[i]._str,argList[i]._str, retList[i]._str)) { ret = true; break; } diff --git a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java index 0d06a18b096..62639b10684 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java +++ b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; @@ -21,7 +20,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import static java.util.stream.Collectors.toUnmodifiableList; +import static java.util.stream.Collectors.toList; /** * Abstract, immutable list for subclassing with concrete types and domain specific filters. @@ -51,7 +50,7 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte /** Returns a new list which is the result of filtering with the -- possibly negated -- condition. */ public final ListType matching(Predicate<Type> condition) { - return constructor.apply(items.stream().filter(negate ? condition.negate() : condition).collect(toUnmodifiableList()), false); + return constructor.apply(items.stream().filter(negate ? condition.negate() : condition).toList(), false); } /** Returns the first n items in this list, or everything except those if negated. */ @@ -72,7 +71,7 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte /** Returns the union of the two lists. */ public ListType and(ListType others) { - return constructor.apply(Stream.concat(items.stream(), others.asList().stream()).collect(toUnmodifiableList()), false); + return constructor.apply(Stream.concat(items.stream(), others.asList().stream()).toList(), false); } /** Returns the items in this as an immutable list. */ @@ -83,19 +82,19 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte /** Returns the items in this as an immutable list after mapping with the given function. */ public final <OtherType> List<OtherType> mapToList(Function<Type, OtherType> mapper) { - return items.stream().map(mapper).collect(toUnmodifiableList()); + return items.stream().map(mapper).toList(); } /** Returns the items sorted by the given comparator. */ public final ListType sortedBy(Comparator<? super Type> comparator) { - return constructor.apply(items.stream().sorted(comparator).collect(toUnmodifiableList()), false); + return constructor.apply(items.stream().sorted(comparator).toList(), false); } /** Returns the items grouped by the given classifier. */ public final <OtherType> Map<OtherType, ListType> groupingBy(Function<Type, OtherType> classifier) { return items.stream().collect(Collectors.groupingBy(classifier, LinkedHashMap::new, - Collectors.collectingAndThen(toUnmodifiableList(), + Collectors.collectingAndThen(toList(), (list) -> constructor.apply(list, false)))); } diff --git a/vespajlib/src/main/java/com/yahoo/config/ini/Ini.java b/vespajlib/src/main/java/com/yahoo/config/ini/Ini.java new file mode 100644 index 00000000000..db1c3a1c98b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/config/ini/Ini.java @@ -0,0 +1,172 @@ +package com.yahoo.config.ini; + +import com.yahoo.yolean.Exceptions; + +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Scanner; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * Basic <a href="https://en.wikipedia.org/wiki/INI_file">INI file</a> parser. + * + * <p>Supported syntax:</p> + * + * <ul> + * <li>Sections. Surrounded with '[' and ']'</li> + * <li>Optional quoting of values. When quoted, the quote character '"' can be escaped with '\'</li> + * <li>Comments, separate and in-line. Indicated with leading ';' or '#'</li> + * </ul> + * + * <p>Behaviour:</p> + * + * <ul> + * <li>Leading and trailing whitespace is always ignored if the value is unquoted</li> + * <li>Sections are sorted in alphabetic order. The same goes for keys within a section</li> + * <li>Empty string in the parsed Map holds section-less config keys</li> + * <li>Duplicated keys within the same section is an error</li> + * <li>Parsing discards comments</li> + * <li>No limitations on section or key names</li> + * </ul> + * + * @param entries Entries of the INI file, grouped by section. + * + * @author mpolden + */ +public record Ini(SortedMap<String, SortedMap<String, String>> entries) { + + private static final char ESCAPE_C = '\\'; + private static final char QUOTE_C = '"'; + private static final String QUOTE = String.valueOf(QUOTE_C); + + public Ini { + var copy = new TreeMap<>(entries); + copy.replaceAll((k, v) -> Collections.unmodifiableSortedMap(new TreeMap<>(copy.get(k)))); + entries = Collections.unmodifiableSortedMap(copy); + } + + /** Write the text representation of this to given output */ + public void write(OutputStream output) { + PrintStream printer = new PrintStream(output, true); + entries.forEach((section, sectionEntries) -> { + if (!section.isEmpty()) { + printer.printf("[%s]\n", section); + } + sectionEntries.forEach((key, value) -> { + printer.printf("%s = %s\n", key, quote(value)); + }); + if (!section.equals(entries.lastKey())) { + printer.println(); + } + }); + } + + /** Parse an INI configuration from given input */ + public static Ini parse(InputStream input) { + SortedMap<String, SortedMap<String, String>> entries = new TreeMap<>(); + Scanner scanner = new Scanner(input, StandardCharsets.UTF_8); + String section = ""; + int lineNum = 0; + while (scanner.hasNextLine()) { + lineNum++; + String line = scanner.nextLine().trim(); + // Blank line + if (line.isEmpty()) { + continue; + } + // Comment + if (isComment(line)) { + continue; + } + // Section + if (line.startsWith("[") && line.endsWith("]")) { + section = line.substring(1, line.length() - 1); + continue; + } + // Key-value entry + try { + Entry entry = Entry.parse(line); + entries.putIfAbsent(section, new TreeMap<>()); + String prevValue = entries.computeIfAbsent(section, (k) -> new TreeMap<>()) + .put(entry.key, entry.value); + if (prevValue != null) { + throw new IllegalArgumentException("Key '" + entry.key + "' duplicated in section '" + + section + "'"); + } + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid entry on line " + lineNum + ": '" + line + "': " + + Exceptions.toMessageString(e)); + } + } + return new Ini(entries); + } + + private static boolean isComment(String s) { + return s.startsWith(";") || s.startsWith("#"); + } + + private static boolean requiresQuoting(String s) { + return s.isEmpty() || s.contains(QUOTE) || !s.equals(s.trim()); + } + + private static boolean unescapedQuoteAt(int index, String s) { + return s.charAt(index) == QUOTE_C && (index == 0 || s.charAt(index - 1) != ESCAPE_C); + } + + private static String quote(String s) { + if (!requiresQuoting(s)) return s; + StringBuilder sb = new StringBuilder(); + sb.append(QUOTE); + for (int i = 0; i < s.length(); i++) { + if (unescapedQuoteAt(i, s)) { + sb.append(ESCAPE_C); + } + sb.append(s.charAt(i)); + } + sb.append(QUOTE); + return sb.toString(); + } + + private record Entry(String key, String value) { + + static Entry parse(String s) { + int equalIndex = s.indexOf('='); + if (equalIndex < 0) throw new IllegalArgumentException("Expected key=[value]"); + String key = s.substring(0, equalIndex).trim(); + String value = s.substring(equalIndex + 1).trim(); + return new Entry(key, dequote(value)); + } + + private static String dequote(String s) { + boolean quoted = s.startsWith(QUOTE); + int end = s.length(); + boolean closeQuote = false; + for (int i = 0; i < s.length(); i++) { + closeQuote = quoted && i > 0 && unescapedQuoteAt(i, s); + boolean startComment = !quoted && isComment(String.valueOf(s.charAt(i))); + if (closeQuote || startComment) { + end = i; + if (quoted && end < s.length() - 1) { + String trailing = s.substring(end + 1).trim(); + if (!isComment(trailing)) { + throw new IllegalArgumentException("Additional character(s) after end quote at column " + end); + } + } + break; + } + } + if (quoted && !closeQuote) { + throw new IllegalArgumentException("Missing closing quote"); + } + int start = quoted ? 1 : 0; + String value = s.substring(start, end); + return quoted ? value : value.trim(); + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index 92bdfb2b3a4..342aca5fb3d 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -225,8 +225,8 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { void validate() { for (int i = 0; i < labels.length; i++) if (labels[i] == null) - throw new IllegalArgumentException("Missing a label for dimension " + - type.dimensions().get(i).name() + " for " + type); + throw new IllegalArgumentException("Missing a label for dimension '" + + type.dimensions().get(i).name() + "' for " + type); } public TensorAddress build() { diff --git a/vespajlib/src/test/java/com/yahoo/config/ini/IniTest.java b/vespajlib/src/test/java/com/yahoo/config/ini/IniTest.java new file mode 100644 index 00000000000..7900f71d410 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/config/ini/IniTest.java @@ -0,0 +1,101 @@ +package com.yahoo.config.ini; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * @author mpolden + */ +class IniTest { + + @Test + public void parse() { + String example = """ + key1 = no section + [] + key2 = also no section ; in-line comment + ; a comment + # another comment + + [foo] + key3 = "with spaces; and an escaped quote: \\" " # in-line comment + key4 = \\"single leading escaped quote + key1 = leading whitespace unquoted + key2 = " leading whitespace quoted" + key6 = + + [bar] + key1=in section + + [foo] + key5 = quote \\" in the middle + """; + Ini ini = parse(example); + assertEquals(Map.of("", Map.of("key1", "no section", + "key2", "also no section"), + "foo", Map.of("key1", "leading whitespace unquoted", + "key2", " leading whitespace quoted", + "key3", "with spaces; and an escaped quote: \\\" ", + "key4", "\\\"single leading escaped quote", + "key5", "quote \\\" in the middle", + "key6", ""), + "bar", Map.of("key1", "in section")), + ini.entries()); + + String expected = """ + key1 = no section + key2 = also no section + + [bar] + key1 = in section + + [foo] + key1 = leading whitespace unquoted + key2 = " leading whitespace quoted" + key3 = "with spaces; and an escaped quote: \\" " + key4 = "\\"single leading escaped quote" + key5 = "quote \\" in the middle" + key6 = "" + """; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ini.write(out); + String serialized = out.toString(StandardCharsets.UTF_8); + assertEquals(expected, serialized); + assertEquals(ini, parse(serialized)); + } + + @Test + public void parse_invalid() { + var tests = Map.of("key1\n", + "Invalid entry on line 1: 'key1': Expected key=[value]", + + "key0 = ok\nkey1 = \"foo bar\" trailing stuff\n", + "Invalid entry on line 2: 'key1 = \"foo bar\" trailing stuff': Additional character(s) after end quote at column 8", + + "[section1]\nkey0=foo\nkey0=bar\n", + "Invalid entry on line 3: 'key0=bar': Key 'key0' duplicated in section 'section1'", + + "key1 = \"foo", + "Invalid entry on line 1: 'key1 = \"foo': Missing closing quote"); + tests.forEach((input, errorMessage) -> { + try { + parse(input); + fail("Expected exception for input '" + input + "'"); + } catch (IllegalArgumentException e) { + assertEquals(errorMessage, e.getMessage()); + } + }); + } + + private static Ini parse(String ini) { + return Ini.parse(new ByteArrayInputStream(ini.getBytes(StandardCharsets.UTF_8))); + } + +} diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 69bd709c613..609c825dafa 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -101,6 +101,7 @@ vespa_define_module( src/tests/net/socket_spec src/tests/net/sync_crypto_socket src/tests/net/tls/auto_reloading_tls_crypto_engine + src/tests/net/tls/capabilities src/tests/net/tls/direct_buffer_bio src/tests/net/tls/openssl_impl src/tests/net/tls/policy_checking_certificate_verifier diff --git a/vespalib/src/tests/memory/memory_test.cpp b/vespalib/src/tests/memory/memory_test.cpp index 1c58da3f663..5345d98cbc7 100644 --- a/vespalib/src/tests/memory/memory_test.cpp +++ b/vespalib/src/tests/memory/memory_test.cpp @@ -168,8 +168,8 @@ TEST("require that Unaligned wrapper works as expected") { Data data; EXPECT_EQUAL(sizeof(Unaligned<uint32_t>), sizeof(uint32_t)); EXPECT_EQUAL(alignof(Unaligned<uint32_t>), 1u); - Unaligned<uint32_t> *arr = &Unaligned<uint32_t>::at(data.get(0)); - const Unaligned<uint32_t> *carr = &Unaligned<uint32_t>::at(data.cget(0)); + Unaligned<uint32_t> *arr = Unaligned<uint32_t>::ptr(data.get(0)); + const Unaligned<uint32_t> *carr = Unaligned<uint32_t>::ptr(data.cget(0)); Unaligned<uint32_t>::at(data.get(0)).write(123); Unaligned<uint32_t>::at(data.get(1)) = 456; arr[2] = 789; diff --git a/vespalib/src/tests/net/tls/capabilities/CMakeLists.txt b/vespalib/src/tests/net/tls/capabilities/CMakeLists.txt new file mode 100644 index 00000000000..4e366674d36 --- /dev/null +++ b/vespalib/src/tests/net/tls/capabilities/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_net_tls_capabilities_test_app TEST + SOURCES + capabilities_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_net_tls_capabilities_test_app + COMMAND vespalib_net_tls_capabilities_test_app) + diff --git a/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp b/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp new file mode 100644 index 00000000000..4a20200c631 --- /dev/null +++ b/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp @@ -0,0 +1,196 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/net/tls/capability_set.h> +#include <vespa/vespalib/testkit/test_kit.h> + +using namespace vespalib; +using namespace vespalib::net::tls; +using namespace std::string_view_literals; + +TEST("Capability instances are equality comparable") { + auto cap1 = Capability::content_document_api(); + auto cap2 = Capability::content_document_api(); + auto cap3 = Capability::content_storage_api(); + EXPECT_EQUAL(cap1, cap2); + EXPECT_EQUAL(cap2, cap1); + EXPECT_NOT_EQUAL(cap1, cap3); +} + +TEST("CapabilitySet instances are equality comparable") { + const auto cap1 = Capability::content_document_api(); + const auto cap2 = Capability::content_search_api(); + + const auto all_caps = CapabilitySet::make_with_all_capabilities(); + const auto set_12_a = CapabilitySet::of({cap1, cap2}); + const auto set_12_b = CapabilitySet::of({cap1, cap2}); + const auto set_1 = CapabilitySet::of({cap1}); + const auto empty = CapabilitySet::make_empty(); + + EXPECT_EQUAL(all_caps, all_caps); + EXPECT_EQUAL(empty, empty); + EXPECT_EQUAL(set_12_a, set_12_b); + EXPECT_EQUAL(set_12_b, set_12_a); + + EXPECT_NOT_EQUAL(all_caps, empty); + EXPECT_NOT_EQUAL(set_12_a, set_1); + EXPECT_NOT_EQUAL(set_12_a, all_caps); + EXPECT_NOT_EQUAL(set_1, empty); +} + +TEST("Can get underlying name of all Capability instances") { + EXPECT_EQUAL(Capability::content_storage_api().name(), "vespa.content.storage_api"sv); + EXPECT_EQUAL(Capability::content_document_api().name(), "vespa.content.document_api"sv); + EXPECT_EQUAL(Capability::content_search_api().name(), "vespa.content.search_api"sv); + EXPECT_EQUAL(Capability::slobrok_api().name(), "vespa.slobrok.api"sv); + EXPECT_EQUAL(Capability::content_status_pages().name(), "vespa.content.status_pages"sv); + EXPECT_EQUAL(Capability::content_metrics_api().name(), "vespa.content.metrics_api"sv); + EXPECT_EQUAL(Capability::content_cluster_controller_internal_state_api().name(), + "vespa.content.cluster_controller.internal_state_api"sv); +} + +TEST("Capability instances can be stringified") { + EXPECT_EQUAL(Capability::content_storage_api().to_string(), "Capability(vespa.content.storage_api)"); +} + +namespace { + +void check_capability_mapping(const std::string& name, Capability expected) { + auto cap = Capability::find_capability(name); + ASSERT_TRUE(cap.has_value()); + EXPECT_EQUAL(*cap, expected); +} + +void check_capability_set_mapping(const std::string& name, CapabilitySet expected) { + auto caps = CapabilitySet::find_capability_set(name); + ASSERT_TRUE(caps.has_value()); + EXPECT_EQUAL(*caps, expected); +} + +} + +TEST("All known capabilities can be looked up by name") { + check_capability_mapping("vespa.content.storage_api", Capability::content_storage_api()); + check_capability_mapping("vespa.content.document_api", Capability::content_document_api()); + check_capability_mapping("vespa.content.search_api", Capability::content_search_api()); + check_capability_mapping("vespa.slobrok.api", Capability::slobrok_api()); + check_capability_mapping("vespa.content.status_pages", Capability::content_status_pages()); + check_capability_mapping("vespa.content.metrics_api", Capability::content_metrics_api()); + check_capability_mapping("vespa.content.cluster_controller.internal_state_api", + Capability::content_cluster_controller_internal_state_api()); +} + +TEST("Unknown capability name returns nullopt") { + EXPECT_FALSE(Capability::find_capability("vespa.content.stale_cat_memes").has_value()); +} + +TEST("CapabilitySet instances can be stringified") { + EXPECT_EQUAL(CapabilitySet::content_node().to_string(), + "CapabilitySet({vespa.content.storage_api, vespa.content.document_api, vespa.slobrok.api})"); +} + +TEST("All known capability sets can be looked up by name") { + check_capability_set_mapping("vespa.content_node", CapabilitySet::content_node()); + check_capability_set_mapping("vespa.container_node", CapabilitySet::container_node()); + check_capability_set_mapping("vespa.telemetry", CapabilitySet::telemetry()); + check_capability_set_mapping("vespa.cluster_controller_node", CapabilitySet::cluster_controller_node()); + check_capability_set_mapping("vespa.config_server", CapabilitySet::config_server()); +} + +TEST("Unknown capability set name returns nullopt") { + EXPECT_FALSE(CapabilitySet::find_capability_set("vespa.unicorn_launcher").has_value()); +} + +TEST("Resolving a capability set adds all its underlying capabilities") { + CapabilitySet caps; + EXPECT_TRUE(caps.resolve_and_add("vespa.content_node")); + // Slightly suboptimal; this test will fail if the default set of capabilities for vespa.content_node changes. + EXPECT_EQUAL(caps.count(), 3u); + EXPECT_FALSE(caps.empty()); + EXPECT_TRUE(caps.contains(Capability::content_storage_api())); + EXPECT_TRUE(caps.contains(Capability::content_document_api())); + EXPECT_TRUE(caps.contains(Capability::slobrok_api())); + EXPECT_FALSE(caps.contains(Capability::content_search_api())); +} + +TEST("Resolving a single capability adds it to the underlying capabilities") { + CapabilitySet caps; + EXPECT_TRUE(caps.resolve_and_add("vespa.slobrok.api")); + EXPECT_EQUAL(caps.count(), 1u); + EXPECT_FALSE(caps.empty()); + EXPECT_TRUE(caps.contains(Capability::slobrok_api())); + EXPECT_FALSE(caps.contains(Capability::content_storage_api())); +} + +TEST("Resolving an unknown capability set returns false and does not add anything") { + CapabilitySet caps; + EXPECT_FALSE(caps.resolve_and_add("vespa.distributors_evil_twin_with_an_evil_goatee")); + EXPECT_EQUAL(caps.count(), 0u); + EXPECT_TRUE(caps.empty()); +} + +TEST("Resolving multiple capabilities/sets adds union of capabilities") { + CapabilitySet caps; + EXPECT_TRUE(caps.resolve_and_add("vespa.content_node")); // CapabilitySet + EXPECT_TRUE(caps.resolve_and_add("vespa.container_node")); // ditto + EXPECT_EQUAL(caps, CapabilitySet::of({Capability::content_storage_api(), Capability::content_document_api(), + Capability::slobrok_api(), Capability::content_search_api()})); + EXPECT_TRUE(caps.resolve_and_add("vespa.content.metrics_api")); // Capability (single) + EXPECT_EQUAL(caps, CapabilitySet::of({Capability::content_storage_api(), Capability::content_document_api(), + Capability::slobrok_api(), Capability::content_search_api(), + Capability::content_metrics_api()})); +} + +TEST("Default-constructed CapabilitySet has no capabilities") { + CapabilitySet caps; + EXPECT_EQUAL(caps.count(), 0u); + EXPECT_TRUE(caps.empty()); + EXPECT_FALSE(caps.contains(Capability::content_storage_api())); +} + +TEST("CapabilitySet can be created with all capabilities") { + auto caps = CapabilitySet::make_with_all_capabilities(); + EXPECT_EQUAL(caps.count(), CapabilitySet::max_count()); + EXPECT_TRUE(caps.contains(Capability::content_storage_api())); + EXPECT_TRUE(caps.contains(Capability::content_metrics_api())); + // ... we just assume the rest are present as well. +} + +TEST("CapabilitySet::contains_all() requires an intersection of capabilities") { + auto cap1 = Capability::content_document_api(); + auto cap2 = Capability::content_search_api(); + auto cap3 = Capability::content_storage_api(); + + const auto all_caps = CapabilitySet::make_with_all_capabilities(); + auto set_123 = CapabilitySet::of({cap1, cap2, cap3}); + auto set_13 = CapabilitySet::of({cap1, cap3}); + auto set_2 = CapabilitySet::of({cap2}); + auto set_23 = CapabilitySet::of({cap2, cap3}); + auto empty = CapabilitySet::make_empty(); + + // Sets contain themselves + EXPECT_TRUE(all_caps.contains_all(all_caps)); + EXPECT_TRUE(set_13.contains_all(set_13)); + EXPECT_TRUE(set_2.contains_all(set_2)); + EXPECT_TRUE(empty.contains_all(empty)); + + // Supersets contain subsets + EXPECT_TRUE(all_caps.contains_all(set_123)); + EXPECT_TRUE(all_caps.contains_all(set_13)); + EXPECT_TRUE(set_123.contains_all(set_13)); + EXPECT_TRUE(set_2.contains_all(empty)); + + // Subsets do not contain supersets + EXPECT_FALSE(set_123.contains_all(all_caps)); + EXPECT_FALSE(set_13.contains_all(set_123)); + EXPECT_FALSE(empty.contains_all(set_2)); + + // Partially overlapping sets are not contained in each other + EXPECT_FALSE(set_13.contains_all(set_23)); + EXPECT_FALSE(set_23.contains_all(set_13)); + + // Fully disjoint sets are not contained in each other + EXPECT_FALSE(set_2.contains_all(set_13)); + EXPECT_FALSE(set_13.contains_all(set_2)); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp index 1de10939bea..3d19c335c19 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp +++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp @@ -542,7 +542,7 @@ struct PrintingCertificateCallback : CertificateVerificationCallback { for (auto& dns : peer_creds.dns_sans) { fprintf(stderr, "Got a DNS SAN entry: %s\n", dns.c_str()); } - return VerificationResult::make_authorized_for_all_roles(); + return VerificationResult::make_authorized_with_all_capabilities(); } }; @@ -551,7 +551,7 @@ struct MockCertificateCallback : CertificateVerificationCallback { mutable PeerCredentials creds; // only used in single thread testing context VerificationResult verify(const PeerCredentials& peer_creds) const override { creds = peer_creds; - return VerificationResult::make_authorized_for_all_roles(); + return VerificationResult::make_authorized_with_all_capabilities(); } }; @@ -712,6 +712,29 @@ TEST_F("Server allows client with certificate that DOES match peer policy", Cert EXPECT_TRUE(f.handshake()); } +TEST_F("Authz policy-derived peer capabilities are propagated to CryptoCodec", CertFixture) { + auto server_ck = f.create_ca_issued_peer_cert({}, {{"DNS:hello.world.example.com"}}); + auto authorized = authorized_peers({policy_with({required_san_dns("stale.memes.example.com")}, + CapabilitySet::of({Capability::content_search_api(), + Capability::content_status_pages()})), + policy_with({required_san_dns("fresh.memes.example.com")}, + CapabilitySet::make_with_all_capabilities())}); + f.reset_server_with_cert_opts(server_ck, std::move(authorized)); + auto client_ck = f.create_ca_issued_peer_cert({}, {{"DNS:stale.memes.example.com"}}); + f.reset_client_with_cert_opts(client_ck, AuthorizedPeers::allow_all_authenticated()); + + ASSERT_TRUE(f.handshake()); + + // Note: "inversion" of client <-> server is because the capabilities are that of the _peer_. + auto client_caps = f.server->granted_capabilities(); + auto server_caps = f.client->granted_capabilities(); + // Server (from client's PoV) implicitly has all capabilities since client doesn't specify any policies + EXPECT_EQUAL(server_caps, CapabilitySet::make_with_all_capabilities()); + // Client (from server's PoV) only has capabilities for the rule matching its DNS SAN entry + EXPECT_EQUAL(client_caps, CapabilitySet::of({Capability::content_search_api(), + Capability::content_status_pages()})); +} + void reset_peers_with_server_authz_mode(CertFixture& f, AuthorizationMode authz_mode) { auto ck = f.create_ca_issued_peer_cert({"hello.world.example.com"}, {}); diff --git a/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp b/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp index fa2bc1a2eaf..c456d7e2a5c 100644 --- a/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp +++ b/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp @@ -127,9 +127,9 @@ bool verify(AuthorizedPeers authorized_peers, const PeerCredentials& peer_creds) return verifier->verify(peer_creds).success(); } -AssumedRoles verify_roles(AuthorizedPeers authorized_peers, const PeerCredentials& peer_creds) { +CapabilitySet verify_capabilities(AuthorizedPeers authorized_peers, const PeerCredentials& peer_creds) { auto verifier = create_verify_callback_from(std::move(authorized_peers)); - return verifier->verify(peer_creds).steal_assumed_roles(); + return verifier->verify(peer_creds).granted_capabilities(); } TEST("Default-constructed AuthorizedPeers does not allow all authenticated peers") { @@ -142,14 +142,16 @@ TEST("Specially constructed set of policies allows all authenticated peers") { EXPECT_TRUE(verify(allow_all, creds_with_dns_sans({{"anything.goes"}}))); } -TEST("specially constructed set of policies returns wildcard role set") { +TEST("specially constructed set of policies returns full capability set") { auto allow_all = AuthorizedPeers::allow_all_authenticated(); - EXPECT_EQUAL(verify_roles(allow_all, creds_with_dns_sans({{"anything.goes"}})), AssumedRoles::make_wildcard_role()); + EXPECT_EQUAL(verify_capabilities(allow_all, creds_with_dns_sans({{"anything.goes"}})), + CapabilitySet::make_with_all_capabilities()); } -TEST("policy without explicit role set implicitly returns wildcard role set") { +TEST("policy without explicit capability set implicitly returns full capability set") { auto authorized = authorized_peers({policy_with({required_san_dns("yolo.swag")})}); - EXPECT_EQUAL(verify_roles(authorized, creds_with_dns_sans({{"yolo.swag"}})), AssumedRoles::make_wildcard_role()); + EXPECT_EQUAL(verify_capabilities(authorized, creds_with_dns_sans({{"yolo.swag"}})), + CapabilitySet::make_with_all_capabilities()); } TEST("Non-empty policies do not allow all authenticated peers") { @@ -246,11 +248,11 @@ struct MultiPolicyMatchFixture { }; MultiPolicyMatchFixture::MultiPolicyMatchFixture() - : authorized(authorized_peers({policy_with({required_san_dns("hello.world")}, assumed_roles({"r1"})), - policy_with({required_san_dns("foo.bar")}, assumed_roles({"r2"})), - policy_with({required_san_dns("zoid.berg")}, assumed_roles({"r2", "r3"})), - policy_with({required_san_dns("secret.sauce")}, AssumedRoles::make_wildcard_role()), - policy_with({required_san_uri("zoid://be.rg/")}, assumed_roles({"r4"}))})) + : authorized(authorized_peers({policy_with({required_san_dns("hello.world")}, CapabilitySet::of({cap_1()})), + policy_with({required_san_dns("foo.bar")}, CapabilitySet::of({cap_2()})), + policy_with({required_san_dns("zoid.berg")}, CapabilitySet::of({cap_2(), cap_3()})), + policy_with({required_san_dns("secret.sauce")}, CapabilitySet::make_with_all_capabilities()), + policy_with({required_san_uri("zoid://be.rg/")}, CapabilitySet::of({cap_4()}))})) {} MultiPolicyMatchFixture::~MultiPolicyMatchFixture() = default; @@ -262,32 +264,37 @@ TEST_F("peer verifies if it matches at least 1 policy of multiple", MultiPolicyM EXPECT_TRUE(verify(f.authorized, creds_with_uri_sans({{"zoid://be.rg/"}}))); } -TEST_F("role set is returned for single matched policy", MultiPolicyMatchFixture) { - EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"hello.world"}})), assumed_roles({"r1"})); - EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"foo.bar"}})), assumed_roles({"r2"})); - EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"zoid.berg"}})), assumed_roles({"r2", "r3"})); - EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"secret.sauce"}})), AssumedRoles::make_wildcard_role()); - EXPECT_EQUAL(verify_roles(f.authorized, creds_with_uri_sans({{"zoid://be.rg/"}})), assumed_roles({"r4"})); +TEST_F("capability set is returned for single matched policy", MultiPolicyMatchFixture) { + EXPECT_EQUAL(verify_capabilities(f.authorized, creds_with_dns_sans({{"hello.world"}})), + CapabilitySet::of({cap_1()})); + EXPECT_EQUAL(verify_capabilities(f.authorized, creds_with_dns_sans({{"foo.bar"}})), + CapabilitySet::of({cap_2()})); + EXPECT_EQUAL(verify_capabilities(f.authorized, creds_with_dns_sans({{"zoid.berg"}})), + CapabilitySet::of({cap_2(), cap_3()})); + EXPECT_EQUAL(verify_capabilities(f.authorized, creds_with_dns_sans({{"secret.sauce"}})), + CapabilitySet::make_with_all_capabilities()); + EXPECT_EQUAL(verify_capabilities(f.authorized, creds_with_uri_sans({{"zoid://be.rg/"}})), + CapabilitySet::of({cap_4()})); } TEST_F("peer verifies if it matches multiple policies", MultiPolicyMatchFixture) { EXPECT_TRUE(verify(f.authorized, creds_with_dns_sans({{"hello.world"}, {"zoid.berg"}}))); } -TEST_F("union role set is returned if multiple policies match", MultiPolicyMatchFixture) { - EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"hello.world"}, {"foo.bar"}, {"zoid.berg"}})), - assumed_roles({"r1", "r2", "r3"})); - // Wildcard role is tracked as a distinct role string - EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"hello.world"}, {"foo.bar"}, {"secret.sauce"}})), - assumed_roles({"r1", "r2", "*"})); +TEST_F("union capability set is returned if multiple policies match", MultiPolicyMatchFixture) { + EXPECT_EQUAL(verify_capabilities(f.authorized, creds_with_dns_sans({{"hello.world"}, {"foo.bar"}, {"zoid.berg"}})), + CapabilitySet::of({cap_1(), cap_2(), cap_3()})); + EXPECT_EQUAL(verify_capabilities(f.authorized, creds_with_dns_sans({{"hello.world"}, {"foo.bar"}, {"secret.sauce"}})), + CapabilitySet::make_with_all_capabilities()); } TEST_F("peer must match at least 1 of multiple policies", MultiPolicyMatchFixture) { EXPECT_FALSE(verify(f.authorized, creds_with_dns_sans({{"does.not.exist"}}))); } -TEST_F("empty role set is returned if no policies match", MultiPolicyMatchFixture) { - EXPECT_EQUAL(verify_roles(f.authorized, creds_with_dns_sans({{"does.not.exist"}})), AssumedRoles::make_empty()); +TEST_F("empty capability set is returned if no policies match", MultiPolicyMatchFixture) { + EXPECT_EQUAL(verify_capabilities(f.authorized, creds_with_dns_sans({{"does.not.exist"}})), + CapabilitySet::make_empty()); } TEST("CN requirement without glob pattern is matched as exact string") { @@ -308,62 +315,32 @@ TEST("CN requirement can include glob wildcards") { EXPECT_FALSE(verify(authorized, creds_with_cn("world"))); } -TEST("AssumedRoles by default contains no roles") { - AssumedRoles roles; - EXPECT_TRUE(roles.empty()); - EXPECT_FALSE(roles.can_assume_role("foo")); - auto empty = AssumedRoles::make_empty(); - EXPECT_EQUAL(roles, empty); -} - -TEST("AssumedRoles can be constructed with an explicit set of roles") { - auto roles = AssumedRoles::make_for_roles({"foo", "bar"}); - EXPECT_TRUE(roles.can_assume_role("foo")); - EXPECT_TRUE(roles.can_assume_role("bar")); - EXPECT_FALSE(roles.can_assume_role("baz")); -} - -TEST("AssumedRoles wildcard role can assume any role") { - auto roles = AssumedRoles::make_wildcard_role(); - EXPECT_TRUE(roles.can_assume_role("foo")); - EXPECT_TRUE(roles.can_assume_role("bar")); -} - -TEST("AssumedRolesBuilder builds union set of added roles") { - AssumedRolesBuilder builder; - builder.add_union(AssumedRoles::make_for_roles({"hello", "world"})); - builder.add_union(AssumedRoles::make_for_roles({"hello", "moon"})); - builder.add_union(AssumedRoles::make_for_roles({"goodbye", "moon"})); - auto roles = builder.build_with_move(); - EXPECT_EQUAL(roles, AssumedRoles::make_for_roles({"hello", "goodbye", "moon", "world"})); -} - TEST("VerificationResult is not authorized by default") { VerificationResult result; EXPECT_FALSE(result.success()); - EXPECT_TRUE(result.assumed_roles().empty()); + EXPECT_TRUE(result.granted_capabilities().empty()); } TEST("VerificationResult can be explicitly created as not authorized") { auto result = VerificationResult::make_not_authorized(); EXPECT_FALSE(result.success()); - EXPECT_TRUE(result.assumed_roles().empty()); + EXPECT_TRUE(result.granted_capabilities().empty()); } -TEST("VerificationResult can be pre-authorized for all roles") { - auto result = VerificationResult::make_authorized_for_all_roles(); +TEST("VerificationResult can be pre-authorized with all capabilities") { + auto result = VerificationResult::make_authorized_with_all_capabilities(); EXPECT_TRUE(result.success()); - EXPECT_FALSE(result.assumed_roles().empty()); - EXPECT_TRUE(result.assumed_roles().can_assume_role("foo")); + EXPECT_FALSE(result.granted_capabilities().empty()); + EXPECT_EQUAL(result.granted_capabilities(), CapabilitySet::make_with_all_capabilities()); } -TEST("VerificationResult can be pre-authorized for an explicit set of roles") { - auto result = VerificationResult::make_authorized_for_roles(AssumedRoles::make_for_roles({"elden", "ring"})); +TEST("VerificationResult can be pre-authorized for an explicit set of capabilities") { + auto result = VerificationResult::make_authorized_with_capabilities(CapabilitySet::of({cap_2(), cap_3()})); EXPECT_TRUE(result.success()); - EXPECT_FALSE(result.assumed_roles().empty()); - EXPECT_TRUE(result.assumed_roles().can_assume_role("elden")); - EXPECT_TRUE(result.assumed_roles().can_assume_role("ring")); - EXPECT_FALSE(result.assumed_roles().can_assume_role("O you don't have the right")); + EXPECT_FALSE(result.granted_capabilities().empty()); + EXPECT_TRUE(result.granted_capabilities().contains(cap_2())); + EXPECT_TRUE(result.granted_capabilities().contains(cap_3())); + EXPECT_FALSE(result.granted_capabilities().contains(cap_1())); } // TODO test CN _and_ SAN diff --git a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp index be2c63b03f2..8d49bdbf73d 100644 --- a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp +++ b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp @@ -198,6 +198,68 @@ TEST("unknown fields are ignored at parse-time") { EXPECT_TRUE(read_options_from_json_string(json).get() != nullptr); // And no exception thrown. } +TEST("policy without explicit capabilities implicitly get all capabilities") { + const char* json = R"({ + "required-credentials":[ + {"field": "SAN_DNS", "must-match": "hello.world"} + ] + })"; + EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("hello.world")}, + CapabilitySet::make_with_all_capabilities())}), + parse_policies(json).authorized_peers()); +} + +TEST("specifying a capability set adds all its underlying capabilities") { + const char* json = R"({ + "required-credentials":[ + {"field": "SAN_DNS", "must-match": "*.cool-content-clusters.example" } + ], + "capabilities": ["vespa.content_node"] + })"; + EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, + CapabilitySet::content_node())}), + parse_policies(json).authorized_peers()); +} + +TEST("can specify single leaf capabilities") { + const char* json = R"({ + "required-credentials":[ + {"field": "SAN_DNS", "must-match": "*.cool-content-clusters.example" } + ], + "capabilities": ["vespa.content.metrics_api", "vespa.slobrok.api"] + })"; + EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, + CapabilitySet::of({Capability::content_metrics_api(), + Capability::slobrok_api()}))}), + parse_policies(json).authorized_peers()); +} + +TEST("specifying multiple capability sets adds union of underlying capabilities") { + const char* json = R"({ + "required-credentials":[ + {"field": "SAN_DNS", "must-match": "*.cool-content-clusters.example" } + ], + "capabilities": ["vespa.content_node", "vespa.container_node"] + })"; + CapabilitySet caps; + caps.add_all(CapabilitySet::content_node()); + caps.add_all(CapabilitySet::container_node()); + EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, caps)}), + parse_policies(json).authorized_peers()); +} + +TEST("empty capabilities array is not allowed") { + const char* json = R"({ + "required-credentials":[ + {"field": "SAN_DNS", "must-match": "*.cool-content-clusters.example" } + ], + "capabilities": [] + })"; + EXPECT_EXCEPTION(parse_policies(json), vespalib::IllegalArgumentException, + "\"capabilities\" array must either be not present (implies " + "all capabilities) or contain at least one capability name"); +} + // TODO test parsing of multiple policies TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/data/memorydatastore.cpp b/vespalib/src/vespa/vespalib/data/memorydatastore.cpp index 6951da217c7..f7fa627ea7e 100644 --- a/vespalib/src/vespa/vespalib/data/memorydatastore.cpp +++ b/vespalib/src/vespa/vespalib/data/memorydatastore.cpp @@ -35,7 +35,9 @@ MemoryDataStore::push_back(const void * data, const size_t sz) Reference ref(static_cast<char *>(buf.get()) + _writePos); _writePos += sz; guard.reset(); - memcpy(ref.data(), data, sz); + if (sz > 0) { + memcpy(ref.data(), data, sz); + } return ref; } diff --git a/vespalib/src/vespa/vespalib/net/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/CMakeLists.txt index e3eb32d3775..05c404ec2a7 100644 --- a/vespalib/src/vespa/vespalib/net/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/net/CMakeLists.txt @@ -9,6 +9,7 @@ endif() vespa_add_library(vespalib_vespalib_net OBJECT SOURCES async_resolver.cpp + connection_auth_context.cpp crypto_engine.cpp crypto_socket.cpp selector.cpp diff --git a/vespalib/src/vespa/vespalib/net/connection_auth_context.cpp b/vespalib/src/vespa/vespalib/net/connection_auth_context.cpp new file mode 100644 index 00000000000..5dd41b3b4d5 --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/connection_auth_context.cpp @@ -0,0 +1,21 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "connection_auth_context.h" + +namespace vespalib::net { + +ConnectionAuthContext::ConnectionAuthContext(tls::PeerCredentials peer_credentials, + tls::CapabilitySet capabilities) noexcept + : _peer_credentials(std::move(peer_credentials)), + _capabilities(std::move(capabilities)) +{ +} + +ConnectionAuthContext::ConnectionAuthContext(const ConnectionAuthContext&) = default; +ConnectionAuthContext& ConnectionAuthContext::operator=(const ConnectionAuthContext&) = default; +ConnectionAuthContext::ConnectionAuthContext(ConnectionAuthContext&&) noexcept = default; +ConnectionAuthContext& ConnectionAuthContext::operator=(ConnectionAuthContext&&) noexcept = default; + +ConnectionAuthContext::~ConnectionAuthContext() = default; + +} diff --git a/vespalib/src/vespa/vespalib/net/connection_auth_context.h b/vespalib/src/vespa/vespalib/net/connection_auth_context.h new file mode 100644 index 00000000000..fc9815f8b8e --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/connection_auth_context.h @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +// TODO consider moving out of tls sub-namespace +#include <vespa/vespalib/net/tls/peer_credentials.h> +#include <vespa/vespalib/net/tls/capability_set.h> + +namespace vespalib::net { + +class ConnectionAuthContext { + tls::PeerCredentials _peer_credentials; + tls::CapabilitySet _capabilities; +public: + ConnectionAuthContext(tls::PeerCredentials peer_credentials, + tls::CapabilitySet capabilities) noexcept; + + ConnectionAuthContext(const ConnectionAuthContext&); + ConnectionAuthContext& operator=(const ConnectionAuthContext&); + ConnectionAuthContext(ConnectionAuthContext&&) noexcept; + ConnectionAuthContext& operator=(ConnectionAuthContext&&) noexcept; + + ~ConnectionAuthContext(); + + const tls::PeerCredentials& peer_credentials() const noexcept { return _peer_credentials; } + const tls::CapabilitySet& capabilities() const noexcept { return _capabilities; } +}; + +} diff --git a/vespalib/src/vespa/vespalib/net/crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/crypto_socket.cpp index 8d3116339a3..0ae90be8539 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/crypto_socket.cpp @@ -1,9 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "crypto_socket.h" +#include <vespa/vespalib/net/connection_auth_context.h> namespace vespalib { CryptoSocket::~CryptoSocket() = default; +std::unique_ptr<net::ConnectionAuthContext> +CryptoSocket::make_auth_context() +{ + return std::make_unique<net::ConnectionAuthContext>( + net::tls::PeerCredentials(), + net::tls::CapabilitySet::make_with_all_capabilities()); +} + } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/crypto_socket.h b/vespalib/src/vespa/vespalib/net/crypto_socket.h index a6ed6553bbe..9ae2af08084 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_socket.h +++ b/vespalib/src/vespa/vespalib/net/crypto_socket.h @@ -7,6 +7,8 @@ namespace vespalib { +namespace net { class ConnectionAuthContext; } + /** * Abstraction of a low-level async network socket which can produce * io events and allows encrypting written data and decrypting read @@ -143,6 +145,18 @@ struct CryptoSocket { **/ virtual void drop_empty_buffers() = 0; + /** + * If the underlying transport channel supports authn/authz, + * returns a new ConnectionAuthContext object containing the verified + * credentials of the peer as well as the resulting peer capabilities + * inferred by our own policy matching. + * + * If the underlying transport channel does _not_ support authn/authz + * (such as a plaintext connection) a dummy context is returned which + * offers _all_ capabilities. + */ + [[nodiscard]] virtual std::unique_ptr<net::ConnectionAuthContext> make_auth_context(); + virtual ~CryptoSocket(); }; diff --git a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt index a94d088b6a8..5be2e0d4387 100644 --- a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vespalib_vespalib_net_tls OBJECT SOURCES - assumed_roles.cpp authorization_mode.cpp auto_reloading_tls_crypto_engine.cpp + capability.cpp + capability_set.cpp crypto_codec.cpp crypto_codec_adapter.cpp maybe_tls_crypto_engine.cpp diff --git a/vespalib/src/vespa/vespalib/net/tls/assumed_roles.cpp b/vespalib/src/vespa/vespalib/net/tls/assumed_roles.cpp deleted file mode 100644 index 672458d0024..00000000000 --- a/vespalib/src/vespa/vespalib/net/tls/assumed_roles.cpp +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "assumed_roles.h" -#include <vespa/vespalib/stllike/asciistream.h> -#include <algorithm> -#include <ostream> - -namespace vespalib::net::tls { - -const string AssumedRoles::WildcardRole("*"); - -AssumedRoles::AssumedRoles() = default; - -AssumedRoles::AssumedRoles(RoleSet assumed_roles) - : _assumed_roles(std::move(assumed_roles)) -{} - -AssumedRoles::AssumedRoles(const AssumedRoles&) = default; -AssumedRoles& AssumedRoles::operator=(const AssumedRoles&) = default; -AssumedRoles::AssumedRoles(AssumedRoles&&) noexcept = default; -AssumedRoles& AssumedRoles::operator=(AssumedRoles&&) noexcept = default; -AssumedRoles::~AssumedRoles() = default; - -bool AssumedRoles::can_assume_role(const string& role) const noexcept { - return (_assumed_roles.contains(role) || _assumed_roles.contains(WildcardRole)); -} - -std::vector<string> AssumedRoles::ordered_roles() const { - std::vector<string> roles; - for (const auto& r : _assumed_roles) { - roles.emplace_back(r); - } - std::sort(roles.begin(), roles.end()); - return roles; -} - -bool AssumedRoles::operator==(const AssumedRoles& rhs) const noexcept { - return (_assumed_roles == rhs._assumed_roles); -} - -void AssumedRoles::print(asciistream& os) const { - os << "AssumedRoles(roles: ["; - auto roles = ordered_roles(); - for (size_t i = 0; i < roles.size(); ++i) { - if (i > 0) { - os << ", "; - } - os << roles[i]; - } - os << "])"; -} - -asciistream& operator<<(asciistream& os, const AssumedRoles& res) { - res.print(os); - return os; -} - -std::ostream& operator<<(std::ostream& os, const AssumedRoles& res) { - os << to_string(res); - return os; -} - -string to_string(const AssumedRoles& res) { - asciistream os; - os << res; - return os.str(); -} - -AssumedRoles AssumedRoles::make_for_roles(RoleSet assumed_roles) { - return AssumedRoles(std::move(assumed_roles)); -} - -AssumedRoles AssumedRoles::make_wildcard_role() { - return AssumedRoles(RoleSet({WildcardRole})); -} - -AssumedRoles AssumedRoles::make_empty() { - return {}; -} - -AssumedRolesBuilder::AssumedRolesBuilder() = default; -AssumedRolesBuilder::~AssumedRolesBuilder() = default; - -void AssumedRolesBuilder::add_union(const AssumedRoles& roles) { - // TODO fix hash_set iterator range insert() - for (const auto& role : roles.unordered_roles()) { - _wip_roles.insert(role); - } -} - -AssumedRoles AssumedRolesBuilder::build_with_move() { - return AssumedRoles::make_for_roles(std::move(_wip_roles)); -} - -} - diff --git a/vespalib/src/vespa/vespalib/net/tls/assumed_roles.h b/vespalib/src/vespa/vespalib/net/tls/assumed_roles.h deleted file mode 100644 index 00d800916fd..00000000000 --- a/vespalib/src/vespa/vespalib/net/tls/assumed_roles.h +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/stllike/hash_set.h> -#include <vespa/vespalib/stllike/string.h> -#include <vector> -#include <iosfwd> - -namespace vespalib { class asciistream; } - -namespace vespalib::net::tls { - -/** - * Encapsulates a set of roles that requests over a particular authenticated - * connection can assume, based on the authorization rules it matched during mTLS - * handshaking. - * - * If at least one role is a wildcard ('*') role, the connection can assume _any_ - * possible role. This is the default when no role constraints are specified in - * the TLS configuration file (legacy behavior). However, a default-constructed - * AssumedRoles instance does not allow any roles to be assumed. - */ -class AssumedRoles { -public: - using RoleSet = hash_set<string>; -private: - RoleSet _assumed_roles; - - static const string WildcardRole; - - explicit AssumedRoles(RoleSet assumed_roles); -public: - AssumedRoles(); - AssumedRoles(const AssumedRoles&); - AssumedRoles& operator=(const AssumedRoles&); - AssumedRoles(AssumedRoles&&) noexcept; - AssumedRoles& operator=(AssumedRoles&&) noexcept; - ~AssumedRoles(); - - [[nodiscard]] bool empty() const noexcept { - return _assumed_roles.empty(); - } - - /** - * Returns true iff `role` is present in the role set OR the role set contains - * the special wildcard role. - */ - [[nodiscard]] bool can_assume_role(const string& role) const noexcept; - - [[nodiscard]] const RoleSet& unordered_roles() const noexcept { - return _assumed_roles; - } - - [[nodiscard]] std::vector<string> ordered_roles() const; - - bool operator==(const AssumedRoles& rhs) const noexcept; - - void print(asciistream& os) const; - - static AssumedRoles make_for_roles(RoleSet assumed_roles); - static AssumedRoles make_wildcard_role(); // Allows assuming _all_ possible roles - static AssumedRoles make_empty(); // Matches _no_ possible roles -}; - -asciistream& operator<<(asciistream&, const AssumedRoles&); -std::ostream& operator<<(std::ostream&, const AssumedRoles&); -string to_string(const AssumedRoles&); - -class AssumedRolesBuilder { - AssumedRoles::RoleSet _wip_roles; -public: - AssumedRolesBuilder(); - ~AssumedRolesBuilder(); - - void add_union(const AssumedRoles& roles); - [[nodiscard]] bool empty() const noexcept { return _wip_roles.empty(); } - [[nodiscard]] AssumedRoles build_with_move(); -}; - -} diff --git a/vespalib/src/vespa/vespalib/net/tls/capability.cpp b/vespalib/src/vespa/vespalib/net/tls/capability.cpp new file mode 100644 index 00000000000..64de250e60d --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/tls/capability.cpp @@ -0,0 +1,62 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "capability.h" +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/stllike/asciistream.h> +#include <array> + +namespace vespalib::net::tls { + +namespace { + +using namespace std::string_view_literals; + +// Important: must match 1-1 with CapabilityId values! +constexpr std::array<std::string_view, Capability::max_value_count()> capability_names = { + "vespa.content.storage_api"sv, + "vespa.content.document_api"sv, + "vespa.content.search_api"sv, + "vespa.content.cluster_controller.internal_state_api"sv, + "vespa.slobrok.api"sv, + "vespa.content.status_pages"sv, + "vespa.content.metrics_api"sv, +}; + +} // anon ns + +std::string_view Capability::name() const noexcept { + return capability_names[id_as_idx()]; +} + +string Capability::to_string() const { + asciistream os; + // TODO asciistream should be made std::string_view-aware + os << "Capability(" << stringref(name().data(), name().length()) << ')'; + return os.str(); +} + +std::optional<Capability> Capability::find_capability(const string& cap_name) noexcept { + static const hash_map<string, Capability> name_to_cap({ + {"vespa.content.storage_api", content_storage_api()}, + {"vespa.content.document_api", content_document_api()}, + {"vespa.content.search_api", content_search_api()}, + {"vespa.content.cluster_controller.internal_state_api", content_cluster_controller_internal_state_api()}, + {"vespa.slobrok.api", slobrok_api()}, + {"vespa.content.status_pages", content_status_pages()}, + {"vespa.content.metrics_api", content_metrics_api()}, + }); + auto iter = name_to_cap.find(cap_name); + return (iter != name_to_cap.end()) ? std::optional<Capability>(iter->second) : std::nullopt; +} + +std::ostream& operator<<(std::ostream& os, const Capability& cap) { + os << cap.to_string(); + return os; +} + +asciistream& operator<<(asciistream& os, const Capability& cap) { + os << cap.to_string(); + return os; +} + +} diff --git a/vespalib/src/vespa/vespalib/net/tls/capability.h b/vespalib/src/vespa/vespalib/net/tls/capability.h new file mode 100644 index 00000000000..842e3f3a363 --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/tls/capability.h @@ -0,0 +1,104 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <bitset> +#include <iosfwd> +#include <optional> +#include <string_view> +#include <vector> + +namespace vespalib { class asciistream; } + +namespace vespalib::net::tls { + +/** + * A capability represents the ability to access a distinct service or API + * plane in Vespa (such as the Document API). + * + * Capability instances are intended to be very cheap to pass and store by value. + */ +class Capability { +private: + // Each ID value corresponds to a unique single-bit position. + // These values shall never be exposed outside the running process, i.e. they + // must be possible to change arbitrarily internally across versions. + enum class Id : uint32_t { + ContentStorageApi = 0, // Must start at zero + ContentDocumentApi, + ContentSearchApi, + ContentClusterControllerInternalStateApi, + SlobrokApi, + ContentStatusPages, + ContentMetricsApi, + // When adding a capability ID to the end, max_value_count() MUST be updated + }; +public: + constexpr static size_t max_value_count() noexcept { + // This must refer to the highest possible CapabilityId enum value. + return static_cast<size_t>(Id::ContentMetricsApi) + 1; + } +private: + Id _cap_id; + + friend class CapabilitySet; // CapabilitySet needs to know the raw IDs for bit set bookkeeping + + constexpr Id id() const noexcept { return _cap_id; } + constexpr uint32_t id_as_idx() const noexcept { return static_cast<uint32_t>(_cap_id); } + + constexpr explicit Capability(Id cap_id) noexcept : _cap_id(cap_id) {} + + constexpr static Capability of(Id id) noexcept { + return Capability(id); + } + +public: + Capability() = delete; // Only valid capabilities can be created. + + constexpr bool operator==(const Capability& rhs) const noexcept { + return (_cap_id == rhs._cap_id); + } + + constexpr bool operator!=(const Capability& rhs) const noexcept { + return !(*this == rhs); + } + + std::string_view name() const noexcept; + string to_string() const; + + static std::optional<Capability> find_capability(const string& cap_name) noexcept; + + constexpr static Capability content_storage_api() noexcept { + return Capability(Id::ContentStorageApi); + } + + constexpr static Capability content_document_api() noexcept { + return Capability(Id::ContentDocumentApi); + } + + constexpr static Capability content_search_api() noexcept { + return Capability(Id::ContentSearchApi); + } + + constexpr static Capability content_cluster_controller_internal_state_api() noexcept { + return Capability(Id::ContentClusterControllerInternalStateApi); + } + + constexpr static Capability slobrok_api() noexcept { + return Capability(Id::SlobrokApi); + } + + constexpr static Capability content_status_pages() noexcept { + return Capability(Id::ContentStatusPages); + } + + constexpr static Capability content_metrics_api() noexcept { + return Capability(Id::ContentMetricsApi); + } + +}; + +std::ostream& operator<<(std::ostream&, const Capability& cap); +asciistream& operator<<(asciistream&, const Capability& cap); + +} diff --git a/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp b/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp new file mode 100644 index 00000000000..3663694e31a --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp @@ -0,0 +1,95 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "capability_set.h" +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/stllike/asciistream.h> +#include <cassert> + +namespace vespalib::net::tls { + +string CapabilitySet::to_string() const { + asciistream os; + os << "CapabilitySet({"; + bool emit_comma = false; + for_each_capability([&emit_comma, &os](Capability cap) { + if (emit_comma) { + os << ", "; + } else { + emit_comma = true; + } + // TODO let asciistream and std::string_view play along + os << stringref(cap.name().data(), cap.name().size()); + }); + os << "})"; + return os.str(); +} + +std::optional<CapabilitySet> CapabilitySet::find_capability_set(const string& cap_set_name) noexcept { + static const hash_map<string, CapabilitySet> name_to_cap_set({ + {"vespa.content_node", content_node()}, + {"vespa.container_node", container_node()}, + {"vespa.telemetry", telemetry()}, + {"vespa.cluster_controller_node", cluster_controller_node()}, + {"vespa.config_server", config_server()} + }); + auto iter = name_to_cap_set.find(cap_set_name); + return (iter != name_to_cap_set.end()) ? std::optional<CapabilitySet>(iter->second) : std::nullopt; +} + +bool CapabilitySet::resolve_and_add(const string& set_or_cap_name) noexcept { + if (auto cap_set = find_capability_set(set_or_cap_name)) { + _capability_mask |= cap_set->_capability_mask; + return true; + } else if (auto cap = Capability::find_capability(set_or_cap_name)) { + _capability_mask |= cap_as_bit_set(*cap); + return true; + } + return false; +} + +// Note: the capability set factory functions below are all just using constexpr and/or inline +// functions, so the compiler will happily optimize them to just "return <constant bit pattern>". + +CapabilitySet CapabilitySet::content_node() noexcept { + return CapabilitySet::of({Capability::content_storage_api(), + Capability::content_document_api(), + Capability::slobrok_api()}); +} + +CapabilitySet CapabilitySet::container_node() noexcept { + return CapabilitySet::of({Capability::content_document_api(), + Capability::content_search_api(), + Capability::slobrok_api()}); +} + +CapabilitySet CapabilitySet::telemetry() noexcept { + return CapabilitySet::of({Capability::content_status_pages(), + Capability::content_metrics_api()}); +} + +CapabilitySet CapabilitySet::cluster_controller_node() noexcept { + return CapabilitySet::of({Capability::content_cluster_controller_internal_state_api(), + Capability::slobrok_api()}); +} + +CapabilitySet CapabilitySet::config_server() noexcept { + return CapabilitySet::of({/*TODO define required capabilities*/}); +} + +CapabilitySet CapabilitySet::make_with_all_capabilities() noexcept { + BitSet bit_set; + bit_set.flip(); // All cap bits set + return CapabilitySet(bit_set); +} + +std::ostream& operator<<(std::ostream& os, const CapabilitySet& cap_set) { + os << cap_set.to_string(); + return os; +} + +asciistream& operator<<(asciistream& os, const CapabilitySet& cap_set) { + os << cap_set.to_string(); + return os; +} + +} diff --git a/vespalib/src/vespa/vespalib/net/tls/capability_set.h b/vespalib/src/vespa/vespalib/net/tls/capability_set.h new file mode 100644 index 00000000000..f86b043ee7b --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/tls/capability_set.h @@ -0,0 +1,117 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "capability.h" +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/hash_set.h> +#include <bitset> +#include <initializer_list> +#include <iosfwd> +#include <optional> +#include <vector> + +namespace vespalib { class asciistream; } + +namespace vespalib::net::tls { + +/** + * A CapabilitySet efficiently represents a finite set (possibly empty) of individual + * capabilities and allows for both single and set-based membership tests. + * + * Factory functions are provided for all predefined Vespa capability sets. + * + * CapabilitySet instances are intended to be very cheap to pass and store by value. + */ +class CapabilitySet { + using BitSet = std::bitset<Capability::max_value_count()>; + BitSet _capability_mask; + + constexpr static uint32_t cap_as_bit_pos(const Capability& cap) noexcept { + return cap.id_as_idx(); + } + + constexpr static BitSet cap_as_bit_set(const Capability& cap) noexcept { + static_assert(Capability::max_value_count() <= 32); // Must fit into uint32_t bitmask + return {uint32_t(1) << cap_as_bit_pos(cap)}; + } + + explicit constexpr CapabilitySet(BitSet capabilities) noexcept + : _capability_mask(capabilities) + {} +public: + constexpr CapabilitySet() noexcept = default; + constexpr ~CapabilitySet() = default; + + string to_string() const; + + bool operator==(const CapabilitySet& rhs) const noexcept { + return (_capability_mask == rhs._capability_mask); + } + + [[nodiscard]] bool empty() const noexcept { + return _capability_mask.none(); + } + size_t count() const noexcept { + return _capability_mask.count(); + } + constexpr static size_t max_count() noexcept { + return Capability::max_value_count(); + } + + [[nodiscard]] constexpr bool contains(Capability cap) const noexcept { + return _capability_mask[cap_as_bit_pos(cap)]; + } + [[nodiscard]] bool contains_all(CapabilitySet caps) const noexcept { + return ((_capability_mask & caps._capability_mask) == caps._capability_mask); + } + + void add(const Capability& cap) noexcept { + _capability_mask |= cap_as_bit_set(cap); + } + void add_all(const CapabilitySet& cap_set) noexcept { + _capability_mask |= cap_set._capability_mask; + } + + template <typename Func> + void for_each_capability(Func f) const noexcept(noexcept(f(Capability::content_storage_api()))) { + for (size_t i = 0; i < _capability_mask.size(); ++i) { + if (_capability_mask[i]) { + f(Capability::of(static_cast<Capability::Id>(i))); + } + } + } + + /** + * Since we have two capability naming "tiers", resolving is done in two steps: + * 1. Check if the name matches a known capability _set_ name. If so, add + * all unique capabilities within the set to our own working set. Return true. + * 2. Check if the name matches a known single capability. If so, add that + * capability to our own working set. Return true. + * 3. Otherwise, return false. + */ + [[nodiscard]] bool resolve_and_add(const string& set_or_cap_name) noexcept; + + static std::optional<CapabilitySet> find_capability_set(const string& cap_set_name) noexcept; + + static CapabilitySet of(std::initializer_list<Capability> caps) noexcept { + CapabilitySet set; + for (const auto& cap : caps) { + set._capability_mask |= cap_as_bit_set(cap); + } + return set; + } + + static CapabilitySet content_node() noexcept; + static CapabilitySet container_node() noexcept; + static CapabilitySet telemetry() noexcept; + static CapabilitySet cluster_controller_node() noexcept; + static CapabilitySet config_server() noexcept; + + static CapabilitySet make_with_all_capabilities() noexcept; + static CapabilitySet make_empty() noexcept { return CapabilitySet(); }; +}; + +std::ostream& operator<<(std::ostream&, const CapabilitySet& cap_set); +asciistream& operator<<(asciistream&, const CapabilitySet& cap_set); + +} diff --git a/vespalib/src/vespa/vespalib/net/tls/certificate_verification_callback.h b/vespalib/src/vespa/vespalib/net/tls/certificate_verification_callback.h index f4d8d39206b..c670d54273e 100644 --- a/vespalib/src/vespa/vespalib/net/tls/certificate_verification_callback.h +++ b/vespalib/src/vespa/vespalib/net/tls/certificate_verification_callback.h @@ -22,7 +22,7 @@ struct CertificateVerificationCallback { // and it is signed by a trusted CA. struct AcceptAllPreVerifiedCertificates : CertificateVerificationCallback { VerificationResult verify([[maybe_unused]] const PeerCredentials& peer_creds) const override { - return VerificationResult::make_authorized_for_all_roles(); // yolo + return VerificationResult::make_authorized_with_all_capabilities(); // yolo } }; diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h index fc729d7dd6a..8b2f258199b 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "capability_set.h" #include <vespa/vespalib/net/socket_address.h> #include <memory> @@ -54,7 +55,6 @@ struct DecodeResult { struct TlsContext; struct PeerCredentials; -class AssumedRoles; // TODO move to different namespace, not dependent on TLS? @@ -185,9 +185,9 @@ public: [[nodiscard]] virtual const PeerCredentials& peer_credentials() const noexcept = 0; /** - * Union set of all assumed roles in the peer policy rules that fully matched the peer's credentials. + * Union set of all granted capabilities in the peer policy rules that fully matched the peer's credentials. */ - [[nodiscard]] virtual const AssumedRoles& assumed_roles() const noexcept = 0; + [[nodiscard]] virtual CapabilitySet granted_capabilities() const noexcept = 0; /* * Creates an implementation defined CryptoCodec that provides at least TLSv1.2 diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp index 03170ae0f68..a50acc55bd0 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "crypto_codec_adapter.h" +#include <vespa/vespalib/net/connection_auth_context.h> #include <assert.h> namespace vespalib::net::tls { @@ -208,4 +209,10 @@ CryptoCodecAdapter::drop_empty_buffers() _output.drop_if_empty(); } +std::unique_ptr<net::ConnectionAuthContext> +CryptoCodecAdapter::make_auth_context() +{ + return std::make_unique<net::ConnectionAuthContext>(_codec->peer_credentials(), _codec->granted_capabilities()); +} + } // namespace vespalib::net::tls diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h index 1d5b57dc9b2..4b3c66cbc3c 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec_adapter.h @@ -46,6 +46,7 @@ public: ssize_t flush() override; ssize_t half_close() override; void drop_empty_buffers() override; + std::unique_ptr<net::ConnectionAuthContext> make_auth_context() override; }; } // namespace vespalib::net::tls diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h index 5be2146b349..ca7237bfa9a 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h @@ -4,8 +4,8 @@ #include <vespa/vespalib/crypto/openssl_typedefs.h> #include <vespa/vespalib/net/socket_address.h> #include <vespa/vespalib/net/socket_spec.h> -#include <vespa/vespalib/net/tls/assumed_roles.h> #include <vespa/vespalib/net/tls/crypto_codec.h> +#include <vespa/vespalib/net/tls/capability_set.h> #include <vespa/vespalib/net/tls/peer_credentials.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <memory> @@ -58,7 +58,7 @@ class OpenSslCryptoCodecImpl : public CryptoCodec { std::optional<DeferredHandshakeParams> _deferred_handshake_params; std::optional<HandshakeResult> _deferred_handshake_result; PeerCredentials _peer_credentials; - AssumedRoles _assumed_roles; + CapabilitySet _granted_capabilities; public: ~OpenSslCryptoCodecImpl() override; @@ -103,8 +103,8 @@ public: return _peer_credentials; } - [[nodiscard]] const AssumedRoles& assumed_roles() const noexcept override { - return _assumed_roles; + [[nodiscard]] CapabilitySet granted_capabilities() const noexcept override { + return _granted_capabilities; } const SocketAddress& peer_address() const noexcept { return _peer_address; } @@ -120,8 +120,8 @@ public: void set_peer_credentials(PeerCredentials peer_credentials) { _peer_credentials = std::move(peer_credentials); } - void set_assumed_roles(AssumedRoles assumed_roles) { - _assumed_roles = std::move(assumed_roles); + void set_granted_capabilities(CapabilitySet granted_capabilities) { + _granted_capabilities = granted_capabilities; } private: OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx, diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp index 3810140854b..d7977f6cd2a 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp @@ -488,7 +488,7 @@ bool OpenSslTlsContextImpl::verify_trusted_certificate(::X509_STORE_CTX* store_c // Store away credentials and role set for later use by requests that arrive over this connection. // TODO encapsulate as const shared_ptr to immutable object to better facilitate sharing? codec_impl.set_peer_credentials(std::move(creds)); - codec_impl.set_assumed_roles(authz_result.steal_assumed_roles()); + codec_impl.set_granted_capabilities(authz_result.granted_capabilities()); } catch (std::exception& e) { LOGBT(error, codec_impl.peer_address().ip_address(), "Got exception during certificate verification callback for peer '%s': %s", diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp b/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp index a4e651f3f19..eaa6a8c2298 100644 --- a/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp @@ -123,13 +123,14 @@ PeerPolicy::PeerPolicy() = default; PeerPolicy::PeerPolicy(std::vector<RequiredPeerCredential> required_peer_credentials) : _required_peer_credentials(std::move(required_peer_credentials)), - _assumed_roles(AssumedRoles::make_wildcard_role()) -{} + _granted_capabilities(CapabilitySet::make_with_all_capabilities()) +{ +} PeerPolicy::PeerPolicy(std::vector<RequiredPeerCredential> required_peer_credentials, - AssumedRoles assumed_roles) + CapabilitySet granted_capabilities) : _required_peer_credentials(std::move(required_peer_credentials)), - _assumed_roles(std::move(assumed_roles)) + _granted_capabilities(granted_capabilities) {} PeerPolicy::~PeerPolicy() = default; @@ -170,7 +171,7 @@ std::ostream& operator<<(std::ostream& os, const RequiredPeerCredential& cred) { std::ostream& operator<<(std::ostream& os, const PeerPolicy& policy) { os << "PeerPolicy("; print_joined(os, policy.required_peer_credentials(), ", "); - os << ")"; + os << ", " << policy.granted_capabilities().to_string() << ")"; return os; } diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h index 6eab8c2c9b2..3314e5e4adf 100644 --- a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h +++ b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "assumed_roles.h" +#include "capability_set.h" #include <vespa/vespalib/stllike/string.h> #include <memory> #include <vector> @@ -50,26 +50,26 @@ public: class PeerPolicy { // _All_ credentials must match for the policy itself to match. std::vector<RequiredPeerCredential> _required_peer_credentials; - AssumedRoles _assumed_roles; + CapabilitySet _granted_capabilities; public: PeerPolicy(); - // This policy is created with a wildcard role set, i.e. full access. + // This policy is created with a full capability set, i.e. unrestricted access. explicit PeerPolicy(std::vector<RequiredPeerCredential> required_peer_credentials); PeerPolicy(std::vector<RequiredPeerCredential> required_peer_credentials, - AssumedRoles assumed_roles); + CapabilitySet granted_capabilities); ~PeerPolicy(); bool operator==(const PeerPolicy& rhs) const noexcept { return ((_required_peer_credentials == rhs._required_peer_credentials) && - (_assumed_roles == rhs._assumed_roles)); + (_granted_capabilities == rhs._granted_capabilities)); } [[nodiscard]] const std::vector<RequiredPeerCredential>& required_peer_credentials() const noexcept { return _required_peer_credentials; } - [[nodiscard]] const AssumedRoles& assumed_roles() const noexcept { - return _assumed_roles; + [[nodiscard]] const CapabilitySet& granted_capabilities() const noexcept { + return _granted_capabilities; } }; diff --git a/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp b/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp index 4018e20225e..d9dbca6e808 100644 --- a/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp @@ -71,16 +71,16 @@ PolicyConfiguredCertificateVerifier::~PolicyConfiguredCertificateVerifier() = de VerificationResult PolicyConfiguredCertificateVerifier::verify(const PeerCredentials& peer_creds) const { if (_authorized_peers.allows_all_authenticated()) { - return VerificationResult::make_authorized_for_all_roles(); + return VerificationResult::make_authorized_with_all_capabilities(); } - AssumedRolesBuilder roles_builder; + CapabilitySet caps; for (const auto& policy : _authorized_peers.peer_policies()) { if (matches_all_policy_requirements(peer_creds, policy)) { - roles_builder.add_union(policy.assumed_roles()); + caps.add_all(policy.granted_capabilities()); } } - if (!roles_builder.empty()) { - return VerificationResult::make_authorized_for_roles(roles_builder.build_with_move()); + if (!caps.empty()) { + return VerificationResult::make_authorized_with_capabilities(std::move(caps)); } else { return VerificationResult::make_not_authorized(); } diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_socket.h b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_socket.h index 0d036fdad4b..01d20155bd1 100644 --- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_socket.h +++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_socket.h @@ -7,7 +7,7 @@ namespace vespalib { struct TlsCryptoSocket : public CryptoSocket { - ~TlsCryptoSocket(); + ~TlsCryptoSocket() override; virtual void inject_read_data(const char *buf, size_t len) = 0; }; diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp b/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp index 2e80135813d..94281d3ef41 100644 --- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp @@ -5,6 +5,8 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/io/mapped_file_input.h> #include <vespa/vespalib/data/memory_input.h> +#include <vespa/vespalib/net/tls/capability_set.h> +#include <vespa/vespalib/stllike/hash_set.h> namespace vespalib::net::tls { @@ -24,7 +26,8 @@ namespace vespalib::net::tls { { "field":"CN", "must-match": "*.config.blarg"}, { "field":"SAN_DNS", "must-match": "*.fancy.config.blarg"} ], - "name": "funky config servers" + "name": "funky config servers", + "capabilities": ["vespa.content.coolstuff"] } ] } @@ -68,8 +71,7 @@ RequiredPeerCredential parse_peer_credential(const Inspector& req_entry) { return RequiredPeerCredential(field, std::move(match)); } -PeerPolicy parse_peer_policy(const Inspector& peer_entry) { - auto& creds = peer_entry["required-credentials"]; +std::vector<RequiredPeerCredential> parse_peer_credentials(const Inspector& creds) { if (creds.children() == 0) { throw IllegalArgumentException("\"required-credentials\" array can't be empty (would allow all peers)"); } @@ -77,7 +79,31 @@ PeerPolicy parse_peer_policy(const Inspector& peer_entry) { for (size_t i = 0; i < creds.children(); ++i) { required_creds.emplace_back(parse_peer_credential(creds[i])); } - return PeerPolicy(std::move(required_creds)); + return required_creds; +} + +CapabilitySet parse_capabilities(const Inspector& caps) { + CapabilitySet capabilities; + if (caps.valid() && (caps.children() == 0)) { + throw IllegalArgumentException("\"capabilities\" array must either be not present (implies " + "all capabilities) or contain at least one capability name"); + } else if (caps.valid()) { + for (size_t i = 0; i < caps.children(); ++i) { + // TODO warn if resolve_and_add returns false; means capability is unknown! + (void)capabilities.resolve_and_add(caps[i].asString().make_string()); + } + } else { + // If no capabilities are specified, all are implicitly granted. + // This avoids breaking every legacy mTLS app ever. + capabilities = CapabilitySet::make_with_all_capabilities(); + } + return capabilities; +} + +PeerPolicy parse_peer_policy(const Inspector& peer_entry) { + auto required_creds = parse_peer_credentials(peer_entry["required-credentials"]); + auto capabilities = parse_capabilities(peer_entry["capabilities"]); + return {std::move(required_creds), std::move(capabilities)}; } AuthorizedPeers parse_authorized_peers(const Inspector& authorized_peers) { diff --git a/vespalib/src/vespa/vespalib/net/tls/verification_result.cpp b/vespalib/src/vespa/vespalib/net/tls/verification_result.cpp index e4833f59f47..f1e50d3115e 100644 --- a/vespalib/src/vespa/vespalib/net/tls/verification_result.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/verification_result.cpp @@ -8,8 +8,8 @@ namespace vespalib::net::tls { VerificationResult::VerificationResult() = default; -VerificationResult::VerificationResult(AssumedRoles assumed_roles) - : _assumed_roles(std::move(assumed_roles)) +VerificationResult::VerificationResult(CapabilitySet granted_capabilities) + : _granted_capabilities(std::move(granted_capabilities)) {} VerificationResult::VerificationResult(const VerificationResult&) = default; @@ -23,19 +23,19 @@ void VerificationResult::print(asciistream& os) const { if (!success()) { os << "NOT AUTHORIZED"; } else { - os << _assumed_roles; + os << _granted_capabilities; } os << ')'; } VerificationResult -VerificationResult::make_authorized_for_roles(AssumedRoles assumed_roles) { - return VerificationResult(std::move(assumed_roles)); +VerificationResult::make_authorized_with_capabilities(CapabilitySet granted_capabilities) { + return VerificationResult(std::move(granted_capabilities)); } VerificationResult -VerificationResult::make_authorized_for_all_roles() { - return VerificationResult(AssumedRoles::make_wildcard_role()); +VerificationResult::make_authorized_with_all_capabilities() { + return VerificationResult(CapabilitySet::make_with_all_capabilities()); } VerificationResult diff --git a/vespalib/src/vespa/vespalib/net/tls/verification_result.h b/vespalib/src/vespa/vespalib/net/tls/verification_result.h index 2de89269ba4..92b32ad92f7 100644 --- a/vespalib/src/vespa/vespalib/net/tls/verification_result.h +++ b/vespalib/src/vespa/vespalib/net/tls/verification_result.h @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "assumed_roles.h" +#include "capability_set.h" #include <vespa/vespalib/stllike/string.h> #include <iosfwd> @@ -13,14 +13,14 @@ namespace vespalib::net::tls { * The result of evaluating configured mTLS authorization rules against the * credentials presented by a successfully authenticated peer certificate. * - * This result contains the union set of all roles specified by the matching - * authorization rules. If no rules matched, the set will be empty. The role + * This result contains the union set of all capabilities granted by the matching + * authorization rules. If no rules matched, the set will be empty. The capability * set will also be empty for a default-constructed instance. */ class VerificationResult { - AssumedRoles _assumed_roles; + CapabilitySet _granted_capabilities; - explicit VerificationResult(AssumedRoles assumed_roles); + explicit VerificationResult(CapabilitySet granted_capabilities); public: VerificationResult(); VerificationResult(const VerificationResult&); @@ -29,22 +29,19 @@ public: VerificationResult& operator=(VerificationResult&&) noexcept; ~VerificationResult(); - // Returns true iff at least one assumed role has been granted. + // Returns true iff at least one capability been granted. [[nodiscard]] bool success() const noexcept { - return !_assumed_roles.empty(); + return !_granted_capabilities.empty(); } - [[nodiscard]] const AssumedRoles& assumed_roles() const noexcept { - return _assumed_roles; - } - [[nodiscard]] AssumedRoles steal_assumed_roles() noexcept { - return std::move(_assumed_roles); + [[nodiscard]] const CapabilitySet& granted_capabilities() const noexcept { + return _granted_capabilities; } void print(asciistream& os) const; - static VerificationResult make_authorized_for_roles(AssumedRoles assumed_roles); - static VerificationResult make_authorized_for_all_roles(); + static VerificationResult make_authorized_with_capabilities(CapabilitySet granted_capabilities); + static VerificationResult make_authorized_with_all_capabilities(); static VerificationResult make_not_authorized(); }; diff --git a/vespalib/src/vespa/vespalib/stllike/asciistream.cpp b/vespalib/src/vespa/vespalib/stllike/asciistream.cpp index 6b673363d2d..caacec196bd 100644 --- a/vespalib/src/vespa/vespalib/stllike/asciistream.cpp +++ b/vespalib/src/vespa/vespalib/stllike/asciistream.cpp @@ -429,22 +429,32 @@ printInt(unsigned long long r, char * tmp, uint8_t i) return i; } +unsigned long long normalize(long long v, bool &negative) { + if (v < 0) { + negative = true; + if (v == std::numeric_limits<long long>::min()) { + // according to UBSAN: + // negation of -9223372036854775808 cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself + return v; + } + return -v; + } + return v; +} + } asciistream & -asciistream::operator << (long long v) +asciistream::operator << (long long v_in) { char tmp[72]; uint8_t i(sizeof(tmp)); bool negative(false); - if (v == 0) { + if (v_in == 0) { tmp[--i] = '0'; } else { - if (v < 0) { - v = -v; - negative = true; - } + unsigned long long v = normalize(v_in, negative); switch (_base) { case 2: i = printInt<2>(v, tmp, i); break; diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.h b/vespalib/src/vespa/vespalib/stllike/hash_fun.h index 4900fcd5a2b..daedddbba1b 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_fun.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.h @@ -69,10 +69,10 @@ size_t hashValue(const char *str) noexcept; size_t hashValue(const void *str, size_t sz) noexcept; struct hash_strings { - size_t operator() (const vespalib::string & arg) const noexcept { return hashValue(arg.c_str()); } + size_t operator() (const vespalib::string & arg) const noexcept { return hashValue(arg.data(), arg.size()); } size_t operator() (vespalib::stringref arg) const noexcept { return hashValue(arg.data(), arg.size()); } size_t operator() (const char * arg) const noexcept { return hashValue(arg); } - size_t operator() (const std::string& arg) const noexcept { return hashValue(arg.c_str()); } + size_t operator() (const std::string& arg) const noexcept { return hashValue(arg.data(), arg.size()); } }; template<> struct hash<const char *> : hash_strings { }; diff --git a/vespalib/src/vespa/vespalib/test/peer_policy_utils.cpp b/vespalib/src/vespa/vespalib/test/peer_policy_utils.cpp index 82d7b9ea07b..c139b1391e0 100644 --- a/vespalib/src/vespa/vespalib/test/peer_policy_utils.cpp +++ b/vespalib/src/vespa/vespalib/test/peer_policy_utils.cpp @@ -16,25 +16,29 @@ RequiredPeerCredential required_san_uri(vespalib::stringref pattern) { return {RequiredPeerCredential::Field::SAN_URI, pattern}; } -AssumedRoles assumed_roles(const std::vector<string>& roles) { - // TODO fix hash_set iterator range ctor to make this a one-liner - AssumedRoles::RoleSet role_set; - for (const auto& role : roles) { - role_set.insert(role); - } - return AssumedRoles::make_for_roles(std::move(role_set)); -} - PeerPolicy policy_with(std::vector<RequiredPeerCredential> creds) { return PeerPolicy(std::move(creds)); } -PeerPolicy policy_with(std::vector<RequiredPeerCredential> creds, AssumedRoles roles) { - return {std::move(creds), std::move(roles)}; +PeerPolicy policy_with(std::vector<RequiredPeerCredential> creds, CapabilitySet capabilities) { + return {std::move(creds), std::move(capabilities)}; } AuthorizedPeers authorized_peers(std::vector<PeerPolicy> peer_policies) { return AuthorizedPeers(std::move(peer_policies)); } +Capability cap_1() { + return Capability::content_search_api(); +} +Capability cap_2() { + return Capability::content_storage_api(); +} +Capability cap_3() { + return Capability::content_document_api(); +} +Capability cap_4() { + return Capability::slobrok_api(); +} + } diff --git a/vespalib/src/vespa/vespalib/test/peer_policy_utils.h b/vespalib/src/vespa/vespalib/test/peer_policy_utils.h index 72e9fde20de..5c6a97cc2c3 100644 --- a/vespalib/src/vespa/vespalib/test/peer_policy_utils.h +++ b/vespalib/src/vespa/vespalib/test/peer_policy_utils.h @@ -2,15 +2,20 @@ #pragma once #include <vespa/vespalib/net/tls/peer_policies.h> +#include <vespa/vespalib/net/tls/capability_set.h> namespace vespalib::net::tls { RequiredPeerCredential required_cn(vespalib::stringref pattern); RequiredPeerCredential required_san_dns(vespalib::stringref pattern); RequiredPeerCredential required_san_uri(vespalib::stringref pattern); -AssumedRoles assumed_roles(const std::vector<string>& roles); PeerPolicy policy_with(std::vector<RequiredPeerCredential> creds); -PeerPolicy policy_with(std::vector<RequiredPeerCredential> creds, AssumedRoles roles); +PeerPolicy policy_with(std::vector<RequiredPeerCredential> creds, CapabilitySet capabilities); AuthorizedPeers authorized_peers(std::vector<PeerPolicy> peer_policies); +// Some shortcuts for valid capabilities: +Capability cap_1(); +Capability cap_2(); +Capability cap_3(); +Capability cap_4(); } diff --git a/vespalib/src/vespa/vespalib/util/compress.cpp b/vespalib/src/vespa/vespalib/util/compress.cpp index ed237da1b28..74e7901b74a 100644 --- a/vespalib/src/vespa/vespalib/util/compress.cpp +++ b/vespalib/src/vespa/vespalib/util/compress.cpp @@ -6,19 +6,6 @@ namespace vespalib::compress { -size_t Integer::compressedPositiveLength(uint64_t n) -{ - if (n < (0x1 << 6)) { - return 1; - } else if (n < (0x1 << 14)) { - return 2; - } else if ( n < (0x1 << 30)) { - return 4; - } else { - throw IllegalArgumentException(make_string("Number '%" PRIu64 "' too big, must extend encoding", n)); - } -} - size_t Integer::compressPositive(uint64_t n, void *destination) { uint8_t * d = static_cast<uint8_t *>(destination); @@ -37,24 +24,19 @@ size_t Integer::compressPositive(uint64_t n, void *destination) d[3] = n & 0xff; return 4; } else { - throw IllegalArgumentException(make_string("Number '%" PRIu64 "' too big, must extend encoding", n)); + throw_too_big(n); } } -size_t Integer::compressedLength(int64_t n) -{ - if (n < 0) { - n = -n; - } - if (n < (0x1 << 5)) { - return 1; - } else if (n < (0x1 << 13)) { - return 2; - } else if ( n < (0x1 << 29)) { - return 4; - } else { - throw IllegalArgumentException(make_string("Number '%" PRId64 "' too big, must extend encoding", n)); - } + +void +Integer::throw_too_big(int64_t n) { + throw IllegalArgumentException(make_string("Number '%" PRId64 "' too big, must extend encoding", n)); +} + +void +Integer::throw_too_big(uint64_t n) { + throw IllegalArgumentException(make_string("Number '%" PRIu64 "' too big, must extend encoding", n)); } size_t Integer::compress(int64_t n, void *destination) @@ -78,7 +60,7 @@ size_t Integer::compress(int64_t n, void *destination) d[3] = n & 0xff; return 4; } else { - throw IllegalArgumentException(make_string("Number '%" PRId64 "' too big, must extend encoding", negative ? -n : n)); + throw_too_big(negative ? -n : n); } } diff --git a/vespalib/src/vespa/vespalib/util/compress.h b/vespalib/src/vespa/vespalib/util/compress.h index 8218a0a43db..c525e186103 100644 --- a/vespalib/src/vespa/vespalib/util/compress.h +++ b/vespalib/src/vespa/vespalib/util/compress.h @@ -27,12 +27,35 @@ public: * @param unsigned number to compute compressed size of in bytes. * @return Will return the number of bytes this positive number will require **/ - static size_t compressedPositiveLength(uint64_t n); + static size_t compressedPositiveLength(uint64_t n) { + if (n < (0x1 << 6)) { + return 1; + } else if (n < (0x1 << 14)) { + return 2; + } else if ( n < (0x1 << 30)) { + return 4; + } else { + throw_too_big(n); + } + } /** * @param number to compute compressed size of in bytes. * @return Will return the number of bytes this number will require **/ - static size_t compressedLength(int64_t n); + static size_t compressedLength(int64_t n) { + if (n < 0) { + n = -n; + } + if (n < (0x1 << 5)) { + return 1; + } else if (n < (0x1 << 13)) { + return 2; + } else if ( n < (0x1 << 29)) { + return 4; + } else { + throw_too_big(n); + } + } /** * Will decompress an integer. * @param pointer to buffer. pointer is automatically advanced. @@ -82,6 +105,9 @@ public: } return numbytes; } +private: + [[ noreturn ]] static void throw_too_big(int64_t n); + [[ noreturn ]] static void throw_too_big(uint64_t n); }; } diff --git a/vespalib/src/vespa/vespalib/util/jsonwriter.h b/vespalib/src/vespa/vespalib/util/jsonwriter.h index 57380b46e5c..966a5e4b073 100644 --- a/vespalib/src/vespa/vespalib/util/jsonwriter.h +++ b/vespalib/src/vespa/vespalib/util/jsonwriter.h @@ -75,36 +75,4 @@ public: stringref toString() const; }; -template<typename T> -struct JSONPrinter -{ - static void printJSON(JSONWriter& w, T v) { - w.appendInt64(v); - } -}; - -template<> -struct JSONPrinter<uint64_t> -{ - static void printJSON(JSONWriter& w, uint64_t v) { - w.appendUInt64(v); - } -}; - -template<> -struct JSONPrinter<float> -{ - static void printJSON(JSONWriter& w, float v) { - w.appendDouble(v); - } -}; - -template<> -struct JSONPrinter<double> -{ - static void printJSON(JSONWriter& w, double v) { - w.appendDouble(v); - } -}; - } diff --git a/vespalib/src/vespa/vespalib/util/memory.h b/vespalib/src/vespa/vespalib/util/memory.h index f2ed6ccae3e..64428756e41 100644 --- a/vespalib/src/vespa/vespalib/util/memory.h +++ b/vespalib/src/vespa/vespalib/util/memory.h @@ -51,12 +51,20 @@ public: static_assert(std::is_trivial_v<T>); static_assert(alignof(T) > 1, "value is always aligned"); - constexpr static Unaligned &at(void *ptr) noexcept { - return *reinterpret_cast<Unaligned*>(ptr); + constexpr static Unaligned &at(void *p) noexcept { + return *reinterpret_cast<Unaligned*>(p); } - constexpr static const Unaligned &at(const void *ptr) noexcept { - return *reinterpret_cast<const Unaligned*>(ptr); + constexpr static const Unaligned &at(const void *p) noexcept { + return *reinterpret_cast<const Unaligned*>(p); } + + constexpr static Unaligned *ptr(void *p) noexcept { + return reinterpret_cast<Unaligned*>(p); + } + constexpr static const Unaligned *ptr(const void *p) noexcept { + return reinterpret_cast<const Unaligned*>(p); + } + T read() const noexcept { T value; static_assert(sizeof(_data) == sizeof(value)); diff --git a/vespalib/src/vespa/vespalib/util/shared_string_repo.h b/vespalib/src/vespa/vespalib/util/shared_string_repo.h index 353e4fd1ca4..7ed50bf0858 100644 --- a/vespalib/src/vespa/vespalib/util/shared_string_repo.h +++ b/vespalib/src/vespa/vespalib/util/shared_string_repo.h @@ -215,7 +215,12 @@ private: string_id resolve(vespalib::stringref str) { uint32_t direct_id = try_make_direct_id(str); if (direct_id >= ID_BIAS) { +#pragma GCC diagnostic push +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ == 12 +#pragma GCC diagnostic ignored "-Warray-bounds" +#endif uint64_t full_hash = XXH3_64bits(str.data(), str.size()); +#pragma GCC diagnostic pop uint32_t part = full_hash & PART_MASK; uint32_t local_hash = full_hash >> PART_BITS; uint32_t local_idx = _partitions[part].resolve(AltKey{str, local_hash}); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index a5d2397dadb..9634d2220c0 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -9,9 +9,6 @@ import com.yahoo.vespa.curator.stats.ThreadLockStats; import org.apache.curator.framework.recipes.locks.InterProcessLock; import java.time.Duration; -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.TimeUnit; /** @@ -24,26 +21,17 @@ import java.util.concurrent.TimeUnit; */ public class Lock implements Mutex { - // TODO(hakon): Remove once debugging is done - private final Object monitor = new Object(); - private long nextSequenceNumber = 0; - private final Map<Long, Long> reentriesByThreadId = new HashMap<>(); - private final Instant created = Instant.now(); - private Curator curator; - private final InterProcessLock mutex; private final String lockPath; public Lock(String lockPath, Curator curator) { this(lockPath, curator.createMutex(lockPath)); - this.curator = curator; } /** Public for testing only */ public Lock(String lockPath, InterProcessLock mutex) { this.lockPath = lockPath; this.mutex = mutex; - this.curator = null; } /** Take the lock with the given timeout. This may be called multiple times from the same thread - each matched by a close */ @@ -65,38 +53,7 @@ public class Lock implements Mutex { " to acquire lock '" + lockPath + "'"); } - invoke(+1L, (lockPath, debug) -> threadLockStats.lockAcquired(debug), lockPath); - } - - @FunctionalInterface - private interface BiConsumer2 { - void accept(String lockPath, String debug); - } - - // TODO(hakon): Remove once debugging is unnecessary - private void invoke(long reentryCountDiff, BiConsumer2 consumer, String lockPath) { - long threadId = Thread.currentThread().getId(); - final long sequenceNumber; - final Map<Long, Long> reentriesByThreadIdCopy; - synchronized (monitor) { - sequenceNumber = nextSequenceNumber++; - reentriesByThreadId.merge(threadId, reentryCountDiff, (oldValue, argumentValue) -> { - long sum = oldValue + argumentValue /* == reentryCountDiff */; - if (sum == 0) { - // Remove from map - return null; - } else { - return sum; - } - }); - reentriesByThreadIdCopy = Map.copyOf(reentriesByThreadId); - } - - String debug = "thread " + threadId + " Lock 0x" + Integer.toHexString(System.identityHashCode(this)) + - "@" + created + " Curator 0x" + Integer.toHexString(System.identityHashCode(curator)) + - " lock " + lockPath + " #" + sequenceNumber + - ", reentries by thread ID = " + reentriesByThreadIdCopy; - consumer.accept(lockPath, debug); + threadLockStats.lockAcquired(); } @Override @@ -105,7 +62,7 @@ public class Lock implements Mutex { // Update metrics now before release() to avoid double-counting time in locked state. // The lockPath must be sent down as close() may be invoked in an order not necessarily // equal to the reverse order of acquires. - invoke(-1L, threadLockStats::preRelease, lockPath); + threadLockStats.preRelease(lockPath); try { mutex.release(); threadLockStats.postRelease(lockPath); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockStats.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockStats.java index a37034b7547..0897078b94d 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockStats.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockStats.java @@ -64,28 +64,28 @@ public class LockStats { } /** Must be invoked only after the first and non-reentry acquisition of the lock. */ - void notifyOfThreadHoldingLock(Thread currentThread, String lockPath, String debug) { + void notifyOfThreadHoldingLock(Thread currentThread, String lockPath) { Thread oldThread = lockPathsHeld.put(lockPath, currentThread); if (oldThread != null) { getLockMetrics(lockPath).incrementAcquireWithoutReleaseCount(); logger.warning("Thread " + currentThread.getName() + " reports it has the lock on " + lockPath + ", but thread " + oldThread.getName() + - " has not reported it released the lock. " + debug); + " has not reported it released the lock"); } } /** Must be invoked only before the last and non-reentry release of the lock. */ - void notifyOfThreadReleasingLock(Thread currentThread, String lockPath, String debug) { + void notifyOfThreadReleasingLock(Thread currentThread, String lockPath) { Thread oldThread = lockPathsHeld.remove(lockPath); if (oldThread == null) { getLockMetrics(lockPath).incrementNakedReleaseCount(); logger.warning("Thread " + currentThread.getName() + " is releasing the lock " + lockPath + - ", but nobody owns that lock. " + debug); + ", but nobody owns that lock"); } else if (oldThread != currentThread) { getLockMetrics(lockPath).incrementForeignReleaseCount(); logger.warning("Thread " + currentThread.getName() + " is releasing the lock " + lockPath + ", but it was owned by thread " + - oldThread.getName() + ". " + debug); + oldThread.getName()); } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java index 24305539be5..f0b26e61870 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java @@ -98,7 +98,7 @@ public class ThreadLockStats { } /** Mutable method (see class doc) */ - public void lockAcquired(String debug) { + public void lockAcquired() { withLastLockAttempt(lockAttempt -> { // Note on the order of lockAcquired() vs notifyOfThreadHoldingLock(): When the latter is // invoked, other threads may query e.g. isAcquired() on the lockAttempt, which would @@ -107,18 +107,18 @@ public class ThreadLockStats { lockAttempt.lockAcquired(); if (!lockAttempt.isReentry()) { - LockStats.getGlobal().notifyOfThreadHoldingLock(thread, lockAttempt.getLockPath(), debug); + LockStats.getGlobal().notifyOfThreadHoldingLock(thread, lockAttempt.getLockPath()); } }); } /** Mutable method (see class doc) */ - public void preRelease(String path, String debug) { + public void preRelease(String path) { withLastLockAttemptFor(path, lockAttempt -> { // Note on the order of these two statement: Same concerns apply here as in lockAcquired(). if (!lockAttempt.isReentry()) { - LockStats.getGlobal().notifyOfThreadReleasingLock(thread, lockAttempt.getLockPath(), debug); + LockStats.getGlobal().notifyOfThreadReleasingLock(thread, lockAttempt.getLockPath()); } lockAttempt.preRelease(); |
