diff options
351 files changed, 5571 insertions, 1710 deletions
diff --git a/client/js/app/package.json b/client/js/app/package.json index 8ea59b0cc35..030fe818ba6 100644 --- a/client/js/app/package.json +++ b/client/js/app/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "react": "^18", + "react-bootstrap": "^2.4.0", "react-dom": "^18" }, "devDependencies": { @@ -33,7 +34,7 @@ "husky": "^7", "prettier": "2", "pretty-quick": "^3", - "react-router-dom": "^6", + "react-router-dom": "^6.3.0", "vite": "^2" } } diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/AddPropertyButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/AddPropertyButton.jsx new file mode 100644 index 00000000000..47b5a67875b --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Buttons/AddPropertyButton.jsx @@ -0,0 +1,46 @@ +import React, { useContext, useState } from 'react'; +import { QueryInputContext } from '../Contexts/QueryInputContext'; +import SimpleButton from './SimpleButton'; + +export default function AddPropertyButton({ id }) { + const { inputs, setInputs, childMap } = useContext(QueryInputContext); + const [childId, setChildId] = useState(1); + + /** + * Add a child to the input that has the provided id + */ + const addChildProperty = () => { + const newInputs = inputs.slice(); + let currentId = id.substring(0, 1); + let index = newInputs.findIndex((element) => element.id === currentId); //get the index of the root parent + let children = newInputs[index].children; + let parentType = newInputs[index].type; + for (let i = 3; i < id.length + 1; i += 2) { + currentId = id.substring(0, i); + index = children.findIndex((element) => element.id === currentId); + parentType = parentType + '_' + children[index].type; + children = children[index].children; + } + let type = childMap[parentType]; + children.push({ + id: id + '.' + childId, + type: type[Object.keys(type)[0]].name, + typeof: type[Object.keys(type)[0]].type, + input: '', + hasChildren: false, + children: [], + }); + setInputs(newInputs); + setChildId((childId) => childId + 1); + }; + + return ( + <SimpleButton + id={`propb${id}`} + className={'addpropsbutton'} + onClick={addChildProperty} + > + + Add property + </SimpleButton> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/AddQueryInputButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/AddQueryInputButton.jsx new file mode 100644 index 00000000000..8aeaff971bf --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Buttons/AddQueryInputButton.jsx @@ -0,0 +1,48 @@ +import React, { useContext } from 'react'; +import { QueryInputContext } from '../Contexts/QueryInputContext'; +import OverlayTrigger from 'react-bootstrap/OverlayTrigger'; +import Tooltip from 'react-bootstrap/Tooltip'; + +export default function AddQueryInput() { + const { inputs, setInputs, id, setId } = useContext(QueryInputContext); + + /** + * Adds a new element to inputs. + * @param {Event} e Event that happened. + */ + const updateInputs = (e) => { + e.preventDefault(); + setId((id) => id + 1); + setInputs((prevInputs) => [ + ...prevInputs, + { + id: `${id + 1}`, + type: 'yql', + typeof: 'String', + input: '', + hasChildren: false, + children: [], + }, + ]); + }; + + return ( + <OverlayTrigger + placement="right" + delay={{ show: 250, hide: 400 }} + overlay={<Tooltip id="button-tooltip">Add row</Tooltip>} + > + <span> + <button + id="addRow" + className="addRow" + height="0" + width="0" + onClick={updateInputs} + > + + + </button> + </span> + </OverlayTrigger> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/ImageButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/ImageButton.jsx new file mode 100644 index 00000000000..f620146bea5 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Buttons/ImageButton.jsx @@ -0,0 +1,25 @@ +import React from 'react'; + +export default function ImageButton({ + onClick, + children, + className, + id, + image, + height = '15', + width = '15', + style, +}) { + return ( + <button id={id} className={className} onClick={onClick}> + <img + src={image} + height={height} + width={width} + style={style} + alt="Missing" + /> + {children} + </button> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/OverlayImageButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/OverlayImageButton.jsx new file mode 100644 index 00000000000..2cb9c7d6e9a --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Buttons/OverlayImageButton.jsx @@ -0,0 +1,40 @@ +import React from 'react'; +import OverlayTrigger from 'react-bootstrap/OverlayTrigger'; +import Tooltip from 'react-bootstrap/Tooltip'; +import ImageButton from './ImageButton'; + +export default function OverlayImageButton({ + onClick, + children, + className, + id, + image, + height = '15', + width = '15', + style, + tooltip, +}) { + return ( + <OverlayTrigger + placement="right" + delay={{ show: 250, hide: 400 }} + overlay={<Tooltip id="button-tooltip">{tooltip}</Tooltip>} + > + <span> + <ImageButton + id={id} + className={className} + image={image} + height={height} + width={width} + style={style} + onClick={onClick} + > + {children} + </ImageButton> + </span> + </OverlayTrigger> + ); +} + +// diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/PasteJSONButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/PasteJSONButton.jsx new file mode 100644 index 00000000000..73dce637500 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Buttons/PasteJSONButton.jsx @@ -0,0 +1,75 @@ +import React, { useContext, useState } from 'react'; +import ImageButton from './ImageButton'; +import pasteImage from '../../assets/img/paste.svg'; +import { QueryInputContext } from '../Contexts/QueryInputContext'; + +export default function PasteJSONButton() { + const { inputs, setInputs, id, setId, levelZeroParameters, childMap } = + useContext(QueryInputContext); + const [paste, setPaste] = useState(false); + + const handleClick = (e) => { + alert('Button is non-functional'); + // setPaste(true); + // window.addEventListener('paste', handlePaste) + }; + + const handlePaste = (e) => { + setPaste(false); + const pastedData = e.clipboardData.getData('text'); + alert('Converting JSON: \n\n ' + pastedData); + window.removeEventListener('paste', handlePaste); + convertPastedJSON(pastedData); + }; + + const convertPastedJSON = (pastedData) => { + try { + var json = JSON.parse(pastedData); + setId(1); + const newInputs = buildFromJSON(json, id); + setInputs(newInputs); + } catch (error) { + alert('Could not parse JSON, with error-message: \n\n' + error.message); + } + }; + + const buildFromJSON = (json, id) => { + let newInputs = []; + let keys = Object.keys(json); + for (let i = 0; i < keys.length; i++) { + let childId = 1; + let newInput = { id: id, type: keys[i] }; + if (json[keys[i]] === Object) { + newInput['typeof'] = 'Parent'; + newInput['input'] = ''; + newInput['hasChildren'] = true; + let tempId = id + '.' + childId; + childId += 1; + newInput['children'] = buildFromJSON(json[keys[i]], tempId); + } else { + newInput['typeof'] = levelZeroParameters[keys[i]].type; + newInput['input'] = json[keys[i]]; + newInput['hasChildren'] = false; + newInput['children'] = []; + } + setId(id + 1); + console.log(newInput); + newInputs.push(newInput); + } + return newInputs; + }; + + return ( + <> + <ImageButton + id="pasteJSON" + className="pasteJSON" + image={pasteImage} + style={{ marginTop: '-2px', marginRight: '3px' }} + onClick={handleClick} + > + {paste ? 'Press CMD + V' : 'Paste JSON'} + </ImageButton> + </> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/ShowQueryButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/ShowQueryButton.jsx new file mode 100644 index 00000000000..789fc387b38 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Buttons/ShowQueryButton.jsx @@ -0,0 +1,29 @@ +import React, { useContext, useState } from 'react'; +import { QueryContext } from '../Contexts/QueryContext'; +import SimpleButton from './SimpleButton'; + +export default function ShowQueryButton() { + const { query, showQuery, setShowQuery } = useContext(QueryContext); + + const handleClick = () => { + setShowQuery(!showQuery); + }; + + return ( + <> + <SimpleButton className="showJSON" onClick={handleClick}> + Show query JSON + </SimpleButton> + {showQuery && ( + <textarea + id="jsonquery" + className="responsebox" + readOnly + cols="70" + rows="15" + value={query} + ></textarea> + )} + </> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/SimpleButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/SimpleButton.jsx new file mode 100644 index 00000000000..a153c9577e4 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Buttons/SimpleButton.jsx @@ -0,0 +1,9 @@ +import React from 'react'; + +export default function SimpleButton({ onClick, children, className, id }) { + return ( + <button id={id} className={className} onClick={onClick}> + {children} + </button> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryContext.jsx b/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryContext.jsx new file mode 100644 index 00000000000..00644c21d41 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryContext.jsx @@ -0,0 +1,14 @@ +import React, { createContext, useState } from 'react'; + +export const QueryContext = createContext(); + +export const QueryProvider = (prop) => { + const [query, setQuery] = useState(''); + const [showQuery, setShowQuery] = useState(false); + + return ( + <QueryContext.Provider value={{ query, setQuery, showQuery, setShowQuery }}> + {prop.children} + </QueryContext.Provider> + ); +}; diff --git a/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryInputContext.jsx b/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryInputContext.jsx new file mode 100644 index 00000000000..064bf51795b --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryInputContext.jsx @@ -0,0 +1,159 @@ +import React, { useState, createContext } from 'react'; + +export const QueryInputContext = createContext(); + +export const QueryInputProvider = (prop) => { + // This is the id of the newest QueryInput, gets updated each time a new one is added + const [id, setId] = useState(1); + + // These are the methods that can be chosen in a QueryInput + const levelZeroParameters = { + yql: { name: 'yql', type: 'String', hasChildren: false }, + hits: { name: 'hits', type: 'Integer', hasChildren: false }, + offset: { name: 'offset', type: 'Integer', hasChildren: false }, + queryProfile: { name: 'queryProfile', type: 'String', hasChildren: false }, + noCache: { name: 'noCache', type: 'Boolean', hasChildren: false }, + groupingSessionCache: { + name: 'groupingSessionCache', + type: 'Boolean', + hasChildren: false, + }, + searchChain: { name: 'searchChain', type: 'String', hasChildren: false }, + timeout: { name: 'timeout', type: 'Float', hasChildren: false }, + tracelevel: { name: 'tracelevel', type: 'Parent', hasChildren: true }, + traceLevel: { name: 'traceLevel', type: 'Integer', hasChildren: false }, + explainLevel: { name: 'explainLevel', type: 'Integer', hasChildren: false }, + explainlevel: { name: 'explainlevel', type: 'Integer', hasChildren: false }, + model: { name: 'model', type: 'Parent', hasChildren: true }, + ranking: { name: 'ranking', type: 'Parent', hasChildren: true }, + collapse: { name: 'collapse', type: 'Parent', hasChildren: true }, + collapsesize: { name: 'collapsesize', type: 'Integer', hasChildren: false }, + collapsefield: { + name: 'collapsefield', + type: 'String', + hasChildren: false, + }, + presentation: { name: 'presentation', type: 'Parent', hasChildren: true }, + pos: { name: 'pos', type: 'Parent', hasChildren: true }, + streaming: { name: 'streaming', type: 'Parent', hasChildren: true }, + rules: { name: 'rules', type: 'Parent', hasChildren: true }, + recall: { name: 'recall', type: 'List', hasChildren: false }, + user: { name: 'user', type: 'String', hasChildren: false }, + metrics: { name: 'metrics', type: 'Parent', hasChildren: true }, + }; + + // Children of the levelZeroParameters that have child attributes + const childMap = { + collapse: { + summary: { name: 'summary', type: 'String', hasChildren: false }, + }, + metrics: { + ignore: { name: 'ignore', type: 'Boolean', hasChildren: false }, + }, + model: { + defaultIndex: { + name: 'defaultIndex', + type: 'String', + hasChildren: false, + }, + encoding: { name: 'encoding', type: 'String', hasChildren: false }, + language: { name: 'language', type: 'String', hasChildren: false }, + queryString: { name: 'queryString', type: 'String', hasChildren: false }, + restrict: { name: 'restrict', type: 'List', hasChildren: false }, + searchPath: { name: 'searchPath', type: 'String', hasChildren: false }, + sources: { name: 'sources', type: 'List', hasChildren: false }, + type: { name: 'type', type: 'String', hasChildren: false }, + }, + pos: { + ll: { name: 'll', type: 'String', hasChildren: false }, + radius: { name: 'radius', type: 'String', hasChildren: false }, + bb: { name: 'bb', type: 'List', hasChildren: false }, + attribute: { name: 'attribute', type: 'String', hasChildren: false }, + }, + presentation: { + bolding: { name: 'bolding', type: 'Boolean', hasChildren: false }, + format: { name: 'format', type: 'String', hasChildren: false }, + summary: { name: 'summary', type: 'String', hasChildren: false }, + template: { name: 'template', type: 'String', hasChildren: false }, + timing: { name: 'timing', type: 'Boolean', hasChildren: false }, + }, + ranking: { + location: { name: 'location', type: 'String', hasChildren: false }, + features: { name: 'features', type: 'String', hasChildren: false }, + listFeatures: { + name: 'listFeatures', + type: 'Boolean', + hasChildren: false, + }, + profile: { name: 'profile', type: 'String', hasChildren: false }, + properties: { name: 'properties', type: 'String', hasChildren: false }, + sorting: { name: 'sorting', type: 'String', hasChildren: false }, + freshness: { name: 'freshness', type: 'String', hasChildren: false }, + queryCache: { name: 'queryCache', type: 'Boolean', hasChildren: false }, + matchPhase: { name: 'matchPhase', type: 'Parent', hasChildren: true }, + }, + ranking_matchPhase: { + maxHits: { name: 'maxHits', type: 'Long', hasChildren: false }, + attribute: { name: 'attribute', type: 'String', hasChildren: false }, + ascending: { name: 'ascending', type: 'Boolean', hasChildren: false }, + diversity: { name: 'diversity', type: 'Parent', hasChildren: true }, + }, + ranking_matchPhase_diversity: { + attribute: { name: 'attribute', type: 'String', hasChildren: false }, + minGroups: { name: 'minGroups', type: 'Long', hasChildren: false }, + }, + rules: { + off: { name: 'off', type: 'Boolean', hasChildren: false }, + rulebase: { name: 'rulebase', type: 'String', hasChildren: false }, + }, + streaming: { + userid: { name: 'userid', type: 'Integer', hasChildren: false }, + groupname: { name: 'groupname', type: 'String', hasChildren: false }, + selection: { name: 'selection', type: 'String', hasChildren: false }, + priority: { name: 'priority', type: 'String', hasChildren: false }, + maxbucketspervisitor: { + name: 'maxbucketspervisitor', + type: 'Integer', + hasChildren: false, + }, + }, + trace: { + timestamps: { name: 'timestamps', type: 'Boolean', hasChildren: false }, + }, + tracelevel: { + rules: { name: 'rules', type: 'Integer', hasChildren: false }, + }, + }; + + const firstChoice = levelZeroParameters[Object.keys(levelZeroParameters)[0]]; + + const [inputs, setInputs] = useState([ + { + id: '1', + type: firstChoice.name, + typeof: firstChoice.type, + input: '', + hasChildren: false, + children: [], + }, + ]); + + const [selectedItems, setSelectedItems] = useState([]); + + return ( + <QueryInputContext.Provider + value={{ + inputs, + setInputs, + id, + setId, + levelZeroParameters, + childMap, + selectedItems, + setSelectedItems, + }} + > + {prop.children} + </QueryInputContext.Provider> + ); +}; diff --git a/client/js/app/src/app/pages/querybuilder/Components/Contexts/ResponseContext.jsx b/client/js/app/src/app/pages/querybuilder/Components/Contexts/ResponseContext.jsx new file mode 100644 index 00000000000..54f5fc955fd --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Contexts/ResponseContext.jsx @@ -0,0 +1,13 @@ +import React, { createContext, useState } from 'react'; + +export const ResponseContext = createContext(); + +export const ResponseProvider = (prop) => { + const [response, setResponse] = useState(''); + + return ( + <ResponseContext.Provider value={{ response, setResponse }}> + {prop.children} + </ResponseContext.Provider> + ); +}; diff --git a/client/js/app/src/app/pages/querybuilder/Components/Navigation/CustomNavbar.jsx b/client/js/app/src/app/pages/querybuilder/Components/Navigation/CustomNavbar.jsx new file mode 100644 index 00000000000..6ffeae4caed --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Navigation/CustomNavbar.jsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { Navbar, Container, Nav } from 'react-bootstrap'; + +function CustomNavbar() { + return ( + <Navbar collapseOnSelect variant="default" expand="sm"> + <Container> + <Navbar.Brand href="https://vespa.ai"> + Vespa. Big data. Real time. + </Navbar.Brand> + <Navbar.Toggle aria-controls="responsive-navbar-nav" /> + <Navbar.Collapse id="responsive-navbar-nav"> + <Nav> + <Nav.Link href="https://blog.vespa.ai/">Blog</Nav.Link> + <Nav.Link variant="link" href="https://twitter.com/vespaengine"> + Twitter + </Nav.Link> + <Nav.Link variant="link" href="https://docs.vespa.ai"> + Docs + </Nav.Link> + <Nav.Link variant="link" href="https://github.com/vespa-engine"> + GitHub + </Nav.Link> + <Nav.Link + variant="link" + href="https://docs.vespa.ai/en/getting-started.html" + > + Get Started Now + </Nav.Link> + </Nav> + </Navbar.Collapse> + </Container> + </Navbar> + ); +} + +export default CustomNavbar; diff --git a/client/js/app/src/app/pages/querybuilder/Components/Navigation/Footer.jsx b/client/js/app/src/app/pages/querybuilder/Components/Navigation/Footer.jsx new file mode 100644 index 00000000000..b110bce943d --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Navigation/Footer.jsx @@ -0,0 +1,68 @@ +import React from 'react'; +import { Table } from 'react-bootstrap'; + +export default function Footer() { + return ( + <footer> + <Table borderless size="sm"> + <thead className="footer-title"> + <tr> + <th>Resources</th> + <th>Contact</th> + <th>Community</th> + </tr> + </thead> + <tbody> + <tr> + <th> + <a href="https://docs.vespa.ai/en/vespa-quick-start.html"> + Getting Started + </a> + </th> + <th> + <a href="https://twitter.com/vespaengine">Twitter</a> + </th> + <th> + <a href="https://github.com/vespa-engine/vespa/blob/master/CONTRIBUTING.md"> + Contributing + </a> + </th> + </tr> + <tr> + <th> + <a href="https://docs.vespa.ai">Documentation</a> + </th> + <th> + <a href="mailto:info@vespa.ai">info@vespa.ai</a> + </th> + <th> + <a href="https://stackoverflow.com/questions/tagged/vespa"> + Stack Overflow + </a> + </th> + </tr> + <tr> + <th> + <a href="https://github.com/vespa-engine/vespa">Open source</a> + </th> + <th> + <a href="https://github.com/vespa-engine/vespa/issues">Issues</a> + </th> + <th> + <a href="https://gitter.im/vespa-engine/Lobby">Gitter</a> + </th> + </tr> + </tbody> + </Table> + <div className="credits"> + <span>Copyright Yahoo</span> + Licensed under{' '} + <a href="https://github.com/vespa-engine/vespa/blob/master/LICENSE"> + Apache License 2.0 + </a> + , <a href="https://github.com/y7kim/agency-jekyll-theme">Theme</a> by + Rick K. + </div> + </footer> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/Info.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/Info.jsx new file mode 100644 index 00000000000..d921c53a602 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Text/Info.jsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { OverlayTrigger, Popover } from 'react-bootstrap'; +import image from '../../assets/img/information.svg'; + +export default function Info({ + id, + className = 'tip', + height = 15, + width = 15, +}) { + const popover = ( + <Popover id={`inf${id}`}> + <Popover.Header as="h3">Popover right</Popover.Header> + <Popover.Body>Content</Popover.Body> + </Popover> + ); + + return ( + <> + <OverlayTrigger + placement="right" + delay={{ show: 250, hide: 400 }} + overlay={popover} + > + <span> + <img + src={image} + height={height} + width={width} + className="information" + alt="Missing" + /> + </span> + </OverlayTrigger> + </> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/QueryDropDownForm.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/QueryDropDownForm.jsx new file mode 100644 index 00000000000..d9a3417752c --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Text/QueryDropDownForm.jsx @@ -0,0 +1,67 @@ +import { QueryInputContext } from '../Contexts/QueryInputContext'; +import React, { useCallback, useContext, useState } from 'react'; +import SimpleDropDownForm from './SimpleDropDownForm'; + +export default function QueryDropdownForm({ choices, id, child = false }) { + const { + inputs, + setInputs, + levelZeroParameters, + childMap, + selectedItems, + setSelectedItems, + } = useContext(QueryInputContext); + const [choice, setChoice] = useState(); + + /** + * Update the state of inputs to reflect the method chosen from the dropdown. + * If the prevoiusly chosen method had children they are removed. + * @param {Event} e Event containing the new type. + */ + const updateType = (e) => { + e.preventDefault(); + const newType = e.target.value; + const newInputs = inputs.slice(); + let currentId = id.substring(0, 1); + let index = newInputs.findIndex((element) => element.id === currentId); + if (child) { + let parentTypes = newInputs[index].type; + let children = newInputs[index].children; + let childChoices = childMap[parentTypes]; + for (let i = 3; i < id.length; i += 2) { + currentId = id.substring(0, i); + index = children.findIndex((element) => element.id === currentId); + let child = children[index]; + parentTypes = parentTypes + '_' + child.type; + childChoices = childMap[parentTypes]; + children = child.children; + } + index = children.findIndex((element) => element.id === id); + children[index].type = newType; + children[index].hasChildren = childChoices[newType].hasChildren; + children[index].children = []; + children[index].typeof = childChoices[newType].type; + setSelectedItems([...selectedItems, newType]); + } else { + newInputs[index].type = newType; + let hasChildren = levelZeroParameters[newType].hasChildren; + newInputs[index].hasChildren = hasChildren; + newInputs[index].children = []; + newInputs[index].typeof = levelZeroParameters[newType].type; + setSelectedItems([...selectedItems, newType]); + } + setInputs(newInputs); + setChoice(newType); + }; + + //TODO: do not display options that have been chosen + + return ( + <SimpleDropDownForm + id={id} + onChange={updateType} + choices={choices} + value={choice} + ></SimpleDropDownForm> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInput.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInput.jsx new file mode 100644 index 00000000000..c5410ae7c4a --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInput.jsx @@ -0,0 +1,74 @@ +import React, { useContext } from 'react'; +import SimpleButton from '../Buttons/SimpleButton'; +import SimpleForm from './SimpleForm'; +import { OverlayTrigger, Tooltip } from 'react-bootstrap'; +import { QueryInputContext } from '../Contexts/QueryInputContext'; +import QueryDropdownForm from './QueryDropDownForm'; +import AddPropertyButton from '../Buttons/AddPropertyButton'; +import QueryInputChild from './QueryInputChild'; + +export default function QueryInput() { + const { inputs, setInputs, levelZeroParameters, childMap } = + useContext(QueryInputContext); + + function removeRow(id) { + const newList = inputs.filter((item) => item.id !== id); + setInputs(newList); + } + + const updateInput = (e) => { + e.preventDefault(); + const fid = e.target.id.replace('v', ''); + const newInputs = inputs.slice(); + const index = newInputs.findIndex((element) => element.id === fid); + newInputs[index].input = e.target.value; + setInputs(newInputs); + }; + + const setPlaceholder = (id) => { + try { + const index = inputs.findIndex((element) => element.id === id); + return inputs[index].typeof; + } catch (error) { + console.log(error); + } + }; + + const inputList = inputs.map((value) => { + return ( + <div key={value.id} id={value.id} className="queryinput"> + <QueryDropdownForm choices={levelZeroParameters} id={value.id} /> + {value.hasChildren ? ( + <> + <AddPropertyButton id={value.id} /> + <QueryInputChild id={value.id} /> + </> + ) : ( + <SimpleForm + id={`v${value.id}`} + size="30" + onChange={updateInput} + placeholder={setPlaceholder(value.id)} + /> + )} + <OverlayTrigger + placement="right" + delay={{ show: 250, hide: 400 }} + overlay={<Tooltip id="button-tooltip">Remove row</Tooltip>} + > + <span> + <SimpleButton + id={`b${value.id}`} + className="removeRow" + onClick={() => removeRow(value.id)} + children="-" + ></SimpleButton> + </span> + </OverlayTrigger> + <br /> + </div> + ); + }); + + return <>{inputList}</>; +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInputChild.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInputChild.jsx new file mode 100644 index 00000000000..0440d6ef1ba --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInputChild.jsx @@ -0,0 +1,194 @@ +import React, { useContext } from 'react'; +import AddPropertyButton from '../Buttons/AddPropertyButton'; +import { QueryInputContext } from '../Contexts/QueryInputContext'; +import QueryDropdownForm from './QueryDropDownForm'; +import SimpleForm from './SimpleForm'; +import { OverlayTrigger, Tooltip } from 'react-bootstrap'; +import SimpleButton from '../Buttons/SimpleButton'; + +export default function QueryInputChild({ id }) { + const { inputs, setInputs, childMap } = useContext(QueryInputContext); + + let index = inputs.findIndex((element) => element.id === id); + let childArray = inputs[index].children; + let currentTypes = inputs[index].type; + + /** + * Update the state of inputs to reflect what is written into the form. + * @param {Event} e Event containing the new input. + */ + const updateInput = (e) => { + e.preventDefault(); + let newInputs = inputs.slice(); + let iterId = e.target.id.replace('v', ''); + let currentId = iterId.substring(0, 1); + let index = newInputs.findIndex((element) => element.id === currentId); + let children = newInputs[index].children; + const traversedChildren = traverseChildren(iterId, children, ''); + children = traversedChildren.children; + index = children.findIndex((element) => element.id === iterId); + children[index].input = e.target.value; + setInputs(newInputs); + }; + + /** + * Returns a placeholder text for a SimpleForm component. + * @param {String} id The id of the SimpleForm component. + * @returns {String} The placeholder text + */ + const setPlaceHolder = (id) => { + let currentId = id.substring(0, 1); + let index = inputs.findIndex((element) => element.id === currentId); + let combinedType = inputs[index].type; + let children = inputs[index].children; + if (id.length > 3) { + const traversedChildren = traverseChildren(id, children, combinedType); + combinedType = traversedChildren.combinedType; + children = traversedChildren.children; + const currentChoice = childMap[combinedType]; + index = children.findIndex((element) => element.id === id); + combinedType = children[index].type; + return currentChoice[combinedType].type; + } else { + const currentChoice = childMap[combinedType]; + index = children.findIndex((element) => element.id === id); + combinedType = children[index].type; + return currentChoice[combinedType].type; + } + }; + + /** + * Removes the row with the provided id. + * @param {String} id Id of row. + */ + const removeRow = (id) => { + let newInputs = inputs.slice(); + let currentId = id.substring(0, 1); + let index = newInputs.findIndex((element) => element.id === currentId); + let children = newInputs[index].children; + const traversedChildren = traverseChildren(id, children, ''); + index = traversedChildren.children.findIndex((element) => element === id); + traversedChildren.children.splice(index, 1); + setInputs(newInputs); + }; + + /** + * Traverses the children until a child with the provided id is reached. + * @param {String} id Id of the innermost child. + * @param {Array} children Array containing serveral child objects. + * @param {String} combinedType The combined type of all traversed children + * @returns {Object} An object containing the children of the child with the provided id and the combined type. + */ + function traverseChildren(id, children, combinedType) { + let currentId; + let index; + for (let i = 3; i < id.length; i += 2) { + currentId = id.substring(0, i); + index = children.findIndex((element) => element.id === currentId); + combinedType = combinedType + '_' + children[index].type; + children = children[index].children; + } + return { children: children, combinedType: combinedType }; + } + + const inputList = childArray.map((child) => { + return ( + <div key={child.id} id={child.id}> + <QueryDropdownForm + choices={childMap[currentTypes]} + id={child.id} + child={true} + /> + {child.hasChildren ? ( + <> + <AddPropertyButton id={child.id} /> + </> + ) : ( + <SimpleForm + id={`v${child.id}`} + size="30" + onChange={updateInput} + placeholder={setPlaceHolder(child.id)} + /> + )} + <OverlayTrigger + placement="right" + delay={{ show: 250, hide: 400 }} + overlay={<Tooltip id="button-tooltip">Remove row</Tooltip>} + > + <span> + <SimpleButton + id={`b${child.id}`} + className="removeRow" + onClick={() => removeRow(child.id)} + children="-" + ></SimpleButton> + </span> + </OverlayTrigger> + <br /> + <Child + type={currentTypes + '_' + child.type} + child={child} + onChange={updateInput} + placeholder={setPlaceHolder} + removeRow={removeRow} + /> + </div> + ); + }); + + return <>{inputList}</>; +} + +function Child({ child, type, onChange, placeholder, removeRow }) { + const { childMap } = useContext(QueryInputContext); + + const nestedChildren = (child.children || []).map((child) => { + return ( + <div key={child.id}> + <QueryDropdownForm + choices={childMap[type]} + id={child.id} + child={true} + /> + {child.hasChildren ? ( + <> + <AddPropertyButton id={child.id} /> + </> + ) : ( + <SimpleForm + id={`v${child.id}`} + size="30" + onChange={onChange} + placeholder={placeholder(child.id)} + /> + )} + <OverlayTrigger + placement="right" + delay={{ show: 250, hide: 400 }} + overlay={<Tooltip id="button-tooltip">Remove row</Tooltip>} + > + <span> + <SimpleButton + id={`b${child.id}`} + className="removeRow" + onClick={() => removeRow(child.id)} + children="-" + ></SimpleButton> + </span> + </OverlayTrigger> + <br /> + <Child + child={child} + id={child.id} + type={type + '_' + child.type} + onChange={onChange} + placeholder={placeholder} + removeRow={removeRow} + /> + </div> + ); + }); + + return <>{nestedChildren}</>; +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/ResponseBox.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/ResponseBox.jsx new file mode 100644 index 00000000000..08c65238434 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Text/ResponseBox.jsx @@ -0,0 +1,17 @@ +import React, { useContext, useEffect, useState } from 'react'; +import { ResponseContext } from '../Contexts/ResponseContext'; + +export default function ResponseBox() { + const { response } = useContext(ResponseContext); + + return ( + <textarea + id="responsetext" + className="responsebox" + readOnly + cols="70" + rows="25" + value={response} + /> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/SendQuery.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/SendQuery.jsx new file mode 100644 index 00000000000..9ee370de7c9 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Text/SendQuery.jsx @@ -0,0 +1,108 @@ +import React, { useContext, useEffect, useState } from 'react'; +import SimpleDropDownForm from './SimpleDropDownForm'; +import SimpleButton from '../Buttons/SimpleButton'; +import SimpleForm from './SimpleForm'; +import { QueryInputContext } from '../Contexts/QueryInputContext'; +import { ResponseContext } from '../Contexts/ResponseContext'; +import { QueryContext } from '../Contexts/QueryContext'; + +export default function SendQuery() { + const { inputs } = useContext(QueryInputContext); + const { setResponse } = useContext(ResponseContext); + const { showQuery, setQuery } = useContext(QueryContext); + + const messageMethods = { post: { name: 'POST' }, get: { name: 'GET' } }; + const [method, setMethod] = useState(messageMethods.post.name); + const [url, setUrl] = useState('http://localhost:8080/search/'); + + useEffect(() => { + const query = buildJSON(inputs, {}); + setQuery(JSON.stringify(query, undefined, 4)); + }, [showQuery]); + + const updateMethod = (e) => { + e.preventDefault(); + const newMethod = e.target.value; + setMethod(newMethod); + }; + + function handleClick() { + const json = buildJSON(inputs, {}); + send(json); + } + + async function send(json) { + let responses = await fetch(url, { + method: method, + headers: { 'Content-Type': 'application/json;charset=utf-8' }, + body: JSON.stringify(json), + }); + if (responses.ok) { + let result = await responses.json(); + let resultObject = JSON.stringify(result, undefined, 4); + setResponse(resultObject); + } + } + + function buildJSON(inputs, json) { + let queryJson = json; + for (let i = 0; i < inputs.length; i++) { + let current = inputs[i]; + let key = current.type; + if (current.hasChildren) { + let child = {}; + child = buildJSON(current.children, child); + queryJson[key] = child; + } else { + queryJson[key] = parseInput(current.input, current.typeof); + } + } + return queryJson; + } + + function parseInput(input, type) { + switch (type) { + case 'Integer': + case 'Long': + return parseInt(input); + break; + + case 'Float': + return parseFloat(input); + break; + + case 'Boolean': + return input.toLowerCase() === 'true' ? true : false; + break; + + default: + return input; + } + } + + const updateUrl = (e) => { + const newUrl = e.target.value; + setUrl(newUrl); + }; + + return ( + <> + <SimpleDropDownForm + choices={messageMethods} + id="method" + className="methodselector" + onChange={updateMethod} + /> + <SimpleForm + id="url" + className="textbox" + initial={url} + size="30" + onChange={updateUrl} + /> + <SimpleButton id="send" className="button" onClick={handleClick}> + Send + </SimpleButton> + </> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleDropDownForm.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleDropDownForm.jsx new file mode 100644 index 00000000000..01288cea44f --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleDropDownForm.jsx @@ -0,0 +1,38 @@ +import React, { useContext, useEffect } from 'react'; +import { QueryInputContext } from '../Contexts/QueryInputContext'; + +export default function SimpleDropDownForm({ + choices, + id, + className = 'input', + onChange, + value, +}) { + const { selectedItems } = useContext(QueryInputContext); + + //FIXME: using the filtered list to render options results in dropdown not changing the displayed selection to what was actually selected. + let filtered = Object.keys(choices).filter( + (choice) => !selectedItems.includes(choice) + ); + useEffect(() => { + filtered = Object.keys(choices).filter( + (choice) => !selectedItems.includes(choice) + ); + }, [selectedItems]); + + const options = Object.keys(choices).map((choice) => { + return ( + <option className="options" key={choice} value={choices[choice].name}> + {choices[choice].name} + </option> + ); + }); + + return ( + <form id={id}> + <select className={className} id={id} value={value} onChange={onChange}> + {options} + </select> + </form> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleForm.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleForm.jsx new file mode 100644 index 00000000000..bb6aaa13529 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleForm.jsx @@ -0,0 +1,34 @@ +import React from 'react'; +import { useState } from 'react'; + +export default function SimpleForm({ + id, + className = 'propvalue', + initial, + size = '20', + onChange, + placeholder, +}) { + SimpleForm.defaultProps = { + onChange: handleChange, + }; + const [input, setValue] = useState(initial); + + function handleChange(e) { + setValue(e.target.value); + } + + return ( + <form className={className} id={id}> + <input + size={size} + type="text" + id={id} + className={className} + defaultValue={initial} + onChange={onChange} + placeholder={placeholder} + /> + </form> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/TextBox.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/TextBox.jsx new file mode 100644 index 00000000000..022b250da7c --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Text/TextBox.jsx @@ -0,0 +1,9 @@ +import React from 'react'; + +export default function TextBox({ id, className, children }) { + return ( + <p className={className} id={id}> + {children} + </p> + ); +} diff --git a/client/js/app/src/app/pages/querybuilder/assets/img/Vespa-V2.png b/client/js/app/src/app/pages/querybuilder/assets/img/Vespa-V2.png Binary files differnew file mode 100644 index 00000000000..ac87f8e94d0 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/assets/img/Vespa-V2.png diff --git a/client/js/app/src/app/pages/querybuilder/assets/img/VespaIcon.png b/client/js/app/src/app/pages/querybuilder/assets/img/VespaIcon.png Binary files differnew file mode 100644 index 00000000000..33063432c20 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/assets/img/VespaIcon.png diff --git a/client/js/app/src/app/pages/querybuilder/assets/img/copy.svg b/client/js/app/src/app/pages/querybuilder/assets/img/copy.svg new file mode 100644 index 00000000000..eada154413a --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/assets/img/copy.svg @@ -0,0 +1,7 @@ +<?xml version="1.0"?> +<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" id="Capa_1" x="0px" y="0px" viewBox="0 0 488.3 488.3" style="enable-background:new 0 0 488.3 488.3;" xml:space="preserve" width="512px" height="512px" class=""><g><g> + <g> + <path d="M314.25,85.4h-227c-21.3,0-38.6,17.3-38.6,38.6v325.7c0,21.3,17.3,38.6,38.6,38.6h227c21.3,0,38.6-17.3,38.6-38.6V124 C352.75,102.7,335.45,85.4,314.25,85.4z M325.75,449.6c0,6.4-5.2,11.6-11.6,11.6h-227c-6.4,0-11.6-5.2-11.6-11.6V124 c0-6.4,5.2-11.6,11.6-11.6h227c6.4,0,11.6,5.2,11.6,11.6V449.6z" data-original="#000000" class="active-path" data-old_color="#F8F5F5" fill="#FCFCFC"/> + <path d="M401.05,0h-227c-21.3,0-38.6,17.3-38.6,38.6c0,7.5,6,13.5,13.5,13.5s13.5-6,13.5-13.5c0-6.4,5.2-11.6,11.6-11.6h227 c6.4,0,11.6,5.2,11.6,11.6v325.7c0,6.4-5.2,11.6-11.6,11.6c-7.5,0-13.5,6-13.5,13.5s6,13.5,13.5,13.5c21.3,0,38.6-17.3,38.6-38.6 V38.6C439.65,17.3,422.35,0,401.05,0z" data-original="#000000" class="active-path" data-old_color="#F8F5F5" fill="#FCFCFC"/> + </g> +</g></g> </svg> diff --git a/client/js/app/src/app/pages/querybuilder/assets/img/down-arrow.svg b/client/js/app/src/app/pages/querybuilder/assets/img/down-arrow.svg new file mode 100644 index 00000000000..d78d2f9c17f --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/assets/img/down-arrow.svg @@ -0,0 +1 @@ +<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 36 36"><defs><style>.cls-1{clip-path:url(#clip-path);}.cls-2{clip-path:url(#clip-path-2);}.cls-3{clip-path:url(#clip-path-3);}</style><clipPath id="clip-path"><path d="M18,4.09A13.91,13.91,0,1,1,4.09,18,13.91,13.91,0,0,1,18,4.09ZM18,35A17,17,0,1,0,6,30,17,17,0,0,0,18,35Z"/></clipPath><clipPath id="clip-path-2"><rect x="-723.82" y="-837.15" width="1483.64" height="3802.85" fill="#FFFFFF"/></clipPath><clipPath id="clip-path-3"><path d="M26,13.83a1.55,1.55,0,0,0-2.16,0L18,19.75l-5.88-6A1.54,1.54,0,0,0,9.93,16L18,24.18,26.07,16a1.57,1.57,0,0,0,0-2.19Z"/></clipPath></defs><title>down-arrow</title><path d="M18,4.09A13.91,13.91,0,1,1,4.09,18,13.91,13.91,0,0,1,18,4.09ZM18,35A17,17,0,1,0,6,30,17,17,0,0,0,18,35Z"/><g class="cls-1"><rect x="-723.82" y="-837.15" width="1483.64" height="3802.85" fill="#FFFFFF"/><g class="cls-2"><rect x="-4.15" y="-4.15" width="44.3" height="44.3" fill="#FFFFFF"/></g></g><path d="M26,13.83a1.55,1.55,0,0,0-2.16,0L18,19.75l-5.88-6A1.54,1.54,0,0,0,9.93,16L18,24.18,26.07,16a1.57,1.57,0,0,0,0-2.19Z"/><g class="cls-3"><rect x="-723.82" y="-837.15" width="1483.64" height="3802.85" fill="#FFFFFF"/><g class="cls-2"><rect x="4.35" y="8.21" width="27.3" height="21.12" fill="#FFFFFF"/></g></g></svg> diff --git a/client/js/app/src/app/pages/querybuilder/assets/img/features-help.png b/client/js/app/src/app/pages/querybuilder/assets/img/features-help.png Binary files differnew file mode 100644 index 00000000000..65702f8b91f --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/assets/img/features-help.png diff --git a/client/js/app/src/app/pages/querybuilder/assets/img/information.svg b/client/js/app/src/app/pages/querybuilder/assets/img/information.svg new file mode 100644 index 00000000000..da42cf2caf6 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/assets/img/information.svg @@ -0,0 +1,10 @@ +<?xml version="1.0"?> +<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" id="Capa_1" x="0px" y="0px" viewBox="0 0 512 512" style="enable-background:new 0 0 512 512;" xml:space="preserve" width="512px" height="512px"><g><g> + <g> + <g> + <circle cx="256" cy="378.5" r="25" data-original="#000000" class="active-path" data-old_color="#898989" fill="#767474"/> + <path d="M256,0C114.516,0,0,114.497,0,256c0,141.484,114.497,256,256,256c141.484,0,256-114.497,256-256 C512,114.516,397.503,0,256,0z M256,472c-119.377,0-216-96.607-216-216c0-119.377,96.607-216,216-216 c119.377,0,216,96.607,216,216C472,375.377,375.393,472,256,472z" data-original="#000000" class="active-path" data-old_color="#898989" fill="#767474"/> + <path d="M256,128.5c-44.112,0-80,35.888-80,80c0,11.046,8.954,20,20,20s20-8.954,20-20c0-22.056,17.944-40,40-40 c22.056,0,40,17.944,40,40c0,22.056-17.944,40-40,40c-11.046,0-20,8.954-20,20v50c0,11.046,8.954,20,20,20 c11.046,0,20-8.954,20-20v-32.531c34.466-8.903,60-40.26,60-77.469C336,164.388,300.112,128.5,256,128.5z" data-original="#000000" class="active-path" data-old_color="#898989" fill="#767474"/> + </g> + </g> +</g></g> </svg> diff --git a/client/js/app/src/app/pages/querybuilder/assets/img/paste.svg b/client/js/app/src/app/pages/querybuilder/assets/img/paste.svg new file mode 100644 index 00000000000..b2edac680bf --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/assets/img/paste.svg @@ -0,0 +1,6 @@ +<?xml version="1.0"?> +<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" id="Capa_1" x="0px" y="0px" width="512px" height="512px" viewBox="0 0 561 561" style="enable-background:new 0 0 561 561;" xml:space="preserve" class=""><g><g> + <g id="content-paste"> + <path d="M459,51H351.9c-10.2-30.6-38.25-51-71.4-51c-33.15,0-61.2,20.4-71.4,51H102c-28.05,0-51,22.95-51,51v408 c0,28.05,22.95,51,51,51h357c28.05,0,51-22.95,51-51V102C510,73.95,487.05,51,459,51z M280.5,51c15.3,0,25.5,10.2,25.5,25.5 S295.8,102,280.5,102S255,91.8,255,76.5S265.2,51,280.5,51z M459,510H102V102h51v76.5h255V102h51V510z" data-original="#000000" class="active-path" data-old_color="#F9F6F6" fill="#FBF9F9"/> + </g> +</g></g> </svg> diff --git a/client/js/app/src/app/pages/querybuilder/assets/img/reload.svg b/client/js/app/src/app/pages/querybuilder/assets/img/reload.svg new file mode 100644 index 00000000000..c5381f9f232 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/assets/img/reload.svg @@ -0,0 +1,6 @@ +<?xml version="1.0"?> +<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" id="Layer_1" x="0px" y="0px" viewBox="0 0 512 512" style="enable-background:new 0 0 512 512;" xml:space="preserve" width="512px" height="512px" class=""><g><g> + <g> + <path d="M482.195,226.196C482.195,101.471,380.725,0,256.001,0S29.805,101.471,29.805,226.196c0,7.409,6.007,13.416,13.416,13.416 s13.416-6.008,13.416-13.416c0-109.93,89.434-199.363,199.363-199.363s199.363,89.434,199.363,199.363 c0,109.928-89.434,199.362-199.363,199.362h-23.276l33.282-37.255c4.937-5.525,4.458-14.007-1.067-18.944 c-5.525-4.937-14.008-4.457-18.944,1.068l-47.576,53.255c-7.788,8.718-7.788,21.866,0,30.584l47.576,53.255 c2.651,2.968,6.322,4.478,10.01,4.478c3.181,0,6.375-1.126,8.934-3.41c5.526-4.937,6.004-13.419,1.067-18.944l-33.282-37.255 h23.276C380.725,452.39,482.195,350.919,482.195,226.196z" data-original="#000000" class="active-path" data-old_color="#F0EDED" fill="#F3F2F2"/> + </g> +</g></g> </svg> diff --git a/client/js/app/src/app/pages/querybuilder/query-builder.jsx b/client/js/app/src/app/pages/querybuilder/query-builder.jsx index fe158fb5b98..472ec1efe49 100644 --- a/client/js/app/src/app/pages/querybuilder/query-builder.jsx +++ b/client/js/app/src/app/pages/querybuilder/query-builder.jsx @@ -1,6 +1,71 @@ -import React from 'react'; -import { Container } from 'app/components'; +import React, { useContext } from 'react'; +import SimpleButton from './Components/Buttons/SimpleButton'; +import QueryInput from './Components/Text/QueryInput'; +import TextBox from './Components/Text/TextBox'; +import ImageButton from './Components/Buttons/ImageButton'; +import OverlayImageButton from './Components/Buttons/OverlayImageButton'; +import AddQueryInput from './Components/Buttons/AddQueryInputButton'; +import { QueryInputProvider } from './Components/Contexts/QueryInputContext'; +import SendQuery from './Components/Text/SendQuery'; +import { + ResponseContext, + ResponseProvider, +} from './Components/Contexts/ResponseContext'; +import ResponseBox from './Components/Text/ResponseBox'; + +import copyImage from './assets/img/copy.svg'; + +import '../../styles/agency.css'; +import '../../styles/vespa.css'; +import ShowQueryButton from './Components/Buttons/ShowQueryButton'; +import { QueryProvider } from './Components/Contexts/QueryContext'; +import PasteJSONButton from './Components/Buttons/PasteJSONButton'; + +//import 'bootstrap/dist/css/bootstrap.min.css'; //TODO: Find out how to get this css export function QueryBuilder() { - return <Container>query builder</Container>; + return ( + <> + <header> + <div className="intro container"> + <TextBox className={'intro-lead-in'}>Vespa Search Engine</TextBox> + <TextBox className={'intro-long'}> + Select the method for sending a request and construct a query. + </TextBox> + <ResponseProvider> + <QueryProvider> + <QueryInputProvider> + <SendQuery /> + <br /> + <div id="request"> + <QueryInput /> + </div> + <br /> + <AddQueryInput /> + <br /> + <PasteJSONButton /> + </QueryInputProvider> + <ShowQueryButton /> + </QueryProvider> + <TextBox className="response">Response</TextBox> + <ResponseBox /> + </ResponseProvider> + <OverlayImageButton + className="intro-copy" + image={copyImage} + height="30" + width="30" + tooltip="Copy" + onClick={() => { + alert('Button is non-functional'); + }} + > + Copy + </OverlayImageButton> + <br /> + <br /> + </div> + </header> + </> + ); } diff --git a/client/js/app/src/app/styles/agency.css b/client/js/app/src/app/styles/agency.css new file mode 100644 index 00000000000..e17809338c6 --- /dev/null +++ b/client/js/app/src/app/styles/agency.css @@ -0,0 +1,895 @@ +/*! + * Start Bootstrap - Agency Bootstrap Theme (http://startbootstrap.com) + * Code licensed under the Apache License v2.0. + * For details, see http://www.apache.org/licenses/LICENSE-2.0. + */ +:root { + --primary: #188fff; + --secondary: #003abc; + --secondary-dark: #333; + --muted: #777; + + --fontprimary: HelveticaNeue, Helvetica, Arial, sans-serif; + --fontsecondary: HelveticaNeue-Thin, Helvetica, Arial, sans-serif; +} + +body { + overflow-x: hidden; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + color: var(--secondary-dark); +} + +.text-muted { + color: var(--muted); +} + +.text-primary { + color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +p { + font-size: 14px; + line-height: 1.75; +} + +p.large { + font-size: 16px; +} + +a, +a:hover, +a:focus, +a:active, +a.active { + outline: 0; +} + +a { + color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +a:hover, +a:focus, +a:active, +a.active { + color: HelveticaNeue-Thin, Helvetica, Arial, sans-serif; +} + +h1, +h2, +h3, +h4, +h5, +h6 { + text-transform: uppercase; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + font-weight: 700; +} + +.img-centered { + margin: 0 auto; +} + +.bg-light-gray { + background-color: #f7f7f7; +} + +.bg-darkest-gray { + background-color: #222; +} + +.btn-primary { + border-color: HelveticaNeue, Helvetica, Arial, sans-serif; + text-transform: uppercase; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + font-weight: 700; + color: #fff; + background-color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +.btn-primary:hover, +.btn-primary:focus, +.btn-primary:active, +.btn-primary.active, +.open .dropdown-toggle.btn-primary { + border-color: HelveticaNeue, Helvetica, Arial, sans-serif; + color: #fff; + background-color: HelveticaNeue-Thin, Helvetica, Arial, sans-serif; +} + +.btn-primary:active, +.btn-primary.active, +.open .dropdown-toggle.btn-primary { + background-image: none; +} + +.btn-primary.disabled, +.btn-primary[disabled], +fieldset[disabled] .btn-primary, +.btn-primary.disabled:hover, +.btn-primary[disabled]:hover, +fieldset[disabled] .btn-primary:hover, +.btn-primary.disabled:focus, +.btn-primary[disabled]:focus, +fieldset[disabled] .btn-primary:focus, +.btn-primary.disabled:active, +.btn-primary[disabled]:active, +fieldset[disabled] .btn-primary:active, +.btn-primary.disabled.active, +.btn-primary[disabled].active, +fieldset[disabled] .btn-primary.active { + border-color: HelveticaNeue, Helvetica, Arial, sans-serif; + background-color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +.btn-primary .badge { + color: HelveticaNeue, Helvetica, Arial, sans-serif; + background-color: #fff; +} + +.btn-xl { + padding: 20px 40px; + border-color: HelveticaNeue, Helvetica, Arial, sans-serif; + border-radius: 3px; + text-transform: uppercase; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + font-size: 18px; + font-weight: 700; + color: #fff; + background-color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +.btn-xl:hover, +.btn-xl:focus, +.btn-xl:active, +.btn-xl.active, +.open .dropdown-toggle.btn-xl { + border-color: HelveticaNeue, Helvetica, Arial, sans-serif; + color: #fff; + background-color: HelveticaNeue-Thin, Helvetica, Arial, sans-serif; +} + +.btn-xl:active, +.btn-xl.active, +.open .dropdown-toggle.btn-xl { + background-image: none; +} + +.btn-xl.disabled, +.btn-xl[disabled], +fieldset[disabled] .btn-xl, +.btn-xl.disabled:hover, +.btn-xl[disabled]:hover, +fieldset[disabled] .btn-xl:hover, +.btn-xl.disabled:focus, +.btn-xl[disabled]:focus, +fieldset[disabled] .btn-xl:focus, +.btn-xl.disabled:active, +.btn-xl[disabled]:active, +fieldset[disabled] .btn-xl:active, +.btn-xl.disabled.active, +.btn-xl[disabled].active, +fieldset[disabled] .btn-xl.active { + border-color: HelveticaNeue, Helvetica, Arial, sans-serif; + background-color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +.btn-xl .badge { + color: HelveticaNeue, Helvetica, Arial, sans-serif; + background-color: #fff; +} + +.navbar-default { + border-color: transparent; + background-color: #222; +} + +.navbar-default .navbar-brand { + font-family: 'Kaushan Script', 'Helvetica Neue', Helvetica, Arial, cursive; + color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +.navbar-default .navbar-brand:hover, +.navbar-default .navbar-brand:focus, +.navbar-default .navbar-brand:active, +.navbar-default .navbar-brand.active { + color: HelveticaNeue-Thin, Helvetica, Arial, sans-serif; +} + +.navbar-default .navbar-collapse { + border-color: rgba(255, 255, 255, 0.02); +} + +.navbar-default .navbar-toggle { + border-color: HelveticaNeue, Helvetica, Arial, sans-serif; + background-color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +.navbar-default .navbar-toggle .icon-bar { + background-color: #fff; +} + +.navbar-default .navbar-toggle:hover, +.navbar-default .navbar-toggle:focus { + background-color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +.navbar-default .nav li a { + text-transform: uppercase; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + font-weight: 400; + letter-spacing: 1px; + color: #fff; + -webkit-transition: all ease 0.35s; + -moz-transition: all ease 0.35s; + transition: all ease 0.35s; +} + +.navbar-default .nav li a:hover, +.navbar-default .nav li a:focus { + outline: 0; + color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +.navbar-default .navbar-nav > .active > a { + border-radius: 0; + color: #fff; + background-color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +.navbar-default .navbar-nav > .active > a:hover, +.navbar-default .navbar-nav > .active > a:focus { + color: #fff; + background-color: HelveticaNeue-Thin, Helvetica, Arial, sans-serif; +} + +@media (min-width: 768px) { + .navbar-default { + padding: 25px 0; + border: 0; + background-color: transparent; + -webkit-transition: padding 0.3s; + -moz-transition: padding 0.3s; + transition: padding 0.3s; + } + + .navbar-default .navbar-brand { + font-size: 2em; + -webkit-transition: all 0.3s; + -moz-transition: all 0.3s; + transition: all 0.3s; + } + + .navbar-default .navbar-nav > .active > a { + border-radius: 3px; + } + + .navbar-default.navbar-shrink { + padding: 10px 0; + background-color: #222; + } + + .navbar-default.navbar-shrink .navbar-brand { + font-size: 1.5em; + } +} + +header { + text-align: center; + color: #fff; + background-attachment: scroll; + /*background-image: url('../img/header-bg.png');*/ + background-position: center center; + background-repeat: none; + -webkit-background-size: cover; + -moz-background-size: cover; + background-size: cover; + -o-background-size: cover; +} + +header .intro-text { + padding-top: 100px; + padding-bottom: 50px; +} + +header .intro-text .intro-lead-in { + margin-bottom: 25px; + font-family: HelveticaNeue-Thin, Helvetica, Arial, sans-serif; + font-size: 22px; + font-style: italic; + line-height: 22px; +} + +header .intro-text .intro-heading { + margin-bottom: 25px; + text-transform: uppercase; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + font-size: 50px; + font-weight: 700; + line-height: 50px; +} + +@media (min-width: 768px) { + header .intro-text { + padding-top: 300px; + padding-bottom: 200px; + } + + header .intro-text .intro-lead-in { + margin-bottom: 25px; + font-family: HelveticaNeue-Thin, Helvetica, Arial, sans-serif; + font-size: 40px; + font-style: italic; + line-height: 40px; + } + + header .intro-text .intro-heading { + margin-bottom: 50px; + text-transform: uppercase; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + font-size: 75px; + font-weight: 700; + line-height: 75px; + } +} + +section { + padding: 100px 0; +} + +section h2.section-heading { + margin-top: 0; + margin-bottom: 15px; + font-size: 40px; +} + +section h3.section-subheading { + margin-bottom: 75px; + text-transform: none; + font-family: HelveticaNeue-Thin, Helvetica, Arial, sans-serif; + font-size: 16px; + font-style: italic; + font-weight: 400; +} + +@media (min-width: 768px) { + section { + padding: 150px 0; + } +} + +.service-heading { + margin: 15px 0; + text-transform: none; +} + +#portfolio .portfolio-item { + right: 0; + margin: 0 0 15px; +} + +#portfolio .portfolio-item .portfolio-link { + display: block; + position: relative; + margin: 0 auto; + max-width: 400px; +} + +#portfolio .portfolio-item .portfolio-link .portfolio-hover { + position: absolute; + width: 100%; + height: 100%; + opacity: 0; + -webkit-transition: all ease 0.5s; + -moz-transition: all ease 0.5s; + transition: all ease 0.5s; + background: rgba(254, 209, 54, 0.9); /* Fallback when no plugin support */ + background: rgba(var(--primary) | hex_to_rgb | join: ','}}, 0.9); +} + +#portfolio .portfolio-item .portfolio-link .portfolio-hover:hover { + opacity: 1; +} + +#portfolio + .portfolio-item + .portfolio-link + .portfolio-hover + .portfolio-hover-content { + position: absolute; + top: 50%; + width: 100%; + height: 20px; + margin-top: -12px; + text-align: center; + font-size: 20px; + color: #fff; +} + +#portfolio + .portfolio-item + .portfolio-link + .portfolio-hover + .portfolio-hover-content + i { + margin-top: -12px; +} + +#portfolio + .portfolio-item + .portfolio-link + .portfolio-hover + .portfolio-hover-content + h3, +#portfolio + .portfolio-item + .portfolio-link + .portfolio-hover + .portfolio-hover-content + h4 { + margin: 0; +} + +#portfolio .portfolio-item .portfolio-caption { + margin: 0 auto; + padding: 25px; + max-width: 400px; + text-align: center; + background-color: #fff; +} + +#portfolio .portfolio-item .portfolio-caption h4 { + margin: 0; + text-transform: none; +} + +#portfolio .portfolio-item .portfolio-caption p { + margin: 0; + font-family: HelveticaNeue-Thin, Helvetica, Arial, sans-serif; + font-size: 16px; + font-style: italic; +} + +#portfolio * { + z-index: 2; +} + +@media (min-width: 767px) { + #portfolio .portfolio-item { + margin: 0 0 30px; + } +} + +.timeline { + position: relative; + padding: 0; + list-style: none; +} + +.timeline:before { + content: ''; + position: absolute; + top: 0; + bottom: 0; + left: 40px; + width: 2px; + margin-left: -1.5px; + background-color: #f1f1f1; +} + +.timeline > li { + position: relative; + margin-bottom: 50px; + min-height: 50px; +} + +.timeline > li:before, +.timeline > li:after { + content: ' '; + display: table; +} + +.timeline > li:after { + clear: both; +} + +.timeline > li .timeline-panel { + float: right; + position: relative; + width: 100%; + padding: 0 20px 0 100px; + text-align: left; +} + +.timeline > li .timeline-panel:before { + right: auto; + left: -15px; + border-right-width: 15px; + border-left-width: 0; +} + +.timeline > li .timeline-panel:after { + right: auto; + left: -14px; + border-right-width: 14px; + border-left-width: 0; +} + +.timeline > li .timeline-image { + z-index: 100; + position: absolute; + left: 0; + width: 80px; + height: 80px; + margin-left: 0; + border: 7px solid #f1f1f1; + border-radius: 100%; + text-align: center; + color: #fff; + background-color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +.timeline > li .timeline-image h4 { + margin-top: 12px; + font-size: 10px; + line-height: 14px; +} + +.timeline > li.timeline-inverted > .timeline-panel { + float: right; + padding: 0 20px 0 100px; + text-align: left; +} + +.timeline > li.timeline-inverted > .timeline-panel:before { + right: auto; + left: -15px; + border-right-width: 15px; + border-left-width: 0; +} + +.timeline > li.timeline-inverted > .timeline-panel:after { + right: auto; + left: -14px; + border-right-width: 14px; + border-left-width: 0; +} + +.timeline > li:last-child { + margin-bottom: 0; +} + +.timeline .timeline-heading h4 { + margin-top: 0; + color: inherit; +} + +.timeline .timeline-heading h4.subheading { + text-transform: none; +} + +.timeline .timeline-body > p, +.timeline .timeline-body > ul { + margin-bottom: 0; +} + +@media (min-width: 768px) { + .timeline:before { + left: 50%; + } + + .timeline > li { + margin-bottom: 100px; + min-height: 100px; + } + + .timeline > li .timeline-panel { + float: left; + width: 41%; + padding: 0 20px 20px 30px; + text-align: right; + } + + .timeline > li .timeline-image { + left: 50%; + width: 100px; + height: 100px; + margin-left: -50px; + } + + .timeline > li .timeline-image h4 { + margin-top: 16px; + font-size: 13px; + line-height: 18px; + } + + .timeline > li.timeline-inverted > .timeline-panel { + float: right; + padding: 0 30px 20px 20px; + text-align: left; + } +} + +@media (min-width: 992px) { + .timeline > li { + min-height: 150px; + } + + .timeline > li .timeline-panel { + padding: 0 20px 20px; + } + + .timeline > li .timeline-image { + width: 150px; + height: 150px; + margin-left: -75px; + } + + .timeline > li .timeline-image h4 { + margin-top: 30px; + font-size: 18px; + line-height: 26px; + } + + .timeline > li.timeline-inverted > .timeline-panel { + padding: 0 20px 20px; + } +} + +@media (min-width: 1200px) { + .timeline > li { + min-height: 170px; + } + + .timeline > li .timeline-panel { + padding: 0 20px 20px 100px; + } + + .timeline > li .timeline-image { + width: 170px; + height: 170px; + margin-left: -85px; + } + + .timeline > li .timeline-image h4 { + margin-top: 40px; + } + + .timeline > li.timeline-inverted > .timeline-panel { + padding: 0 100px 20px 20px; + } +} + +.team-member { + margin-bottom: 50px; + text-align: center; +} + +.team-member img { + margin: 0 auto; + border: 7px solid #fff; +} + +.team-member h4 { + margin-top: 25px; + margin-bottom: 0; + text-transform: none; +} + +.team-member p { + margin-top: 0; +} + +aside.clients img { + margin: 50px auto; +} + +section#contact { + background-color: #222; + /* background-image: url('../img/map-image.png'); */ + background-position: center; + background-repeat: no-repeat; +} + +section#contact .section-heading { + color: #fff; +} + +section#contact .form-group { + margin-bottom: 25px; +} + +section#contact .form-group input, +section#contact .form-group textarea { + padding: 20px; +} + +section#contact .form-group input.form-control { + height: auto; +} + +section#contact .form-group textarea.form-control { + height: 236px; +} + +section#contact .form-control:focus { + border-color: HelveticaNeue, Helvetica, Arial, sans-serif; + box-shadow: none; +} + +section#contact::-webkit-input-placeholder { + text-transform: uppercase; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + font-weight: 700; + color: #bbb; +} + +section#contact:-moz-placeholder { + text-transform: uppercase; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + font-weight: 700; + color: #bbb; +} + +section#contact::-moz-placeholder { + text-transform: uppercase; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + font-weight: 700; + color: #bbb; +} + +section#contact:-ms-input-placeholder { + text-transform: uppercase; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + font-weight: 700; + color: #bbb; +} + +section#contact .text-danger { + color: #e74c3c; +} + +footer { + padding: 25px 0; + text-align: center; +} + +footer span.copyright { + text-transform: uppercase; + text-transform: none; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + line-height: 40px; +} + +footer ul.quicklinks { + margin-bottom: 0; + text-transform: uppercase; + text-transform: none; + font-family: HelveticaNeue, Helvetica, Arial, sans-serif; + line-height: 40px; +} + +ul.social-buttons { + margin-bottom: 0; +} + +ul.social-buttons li a { + display: block; + width: 40px; + height: 40px; + border-radius: 100%; + font-size: 20px; + line-height: 40px; + outline: 0; + color: #fff; + background-color: #222; + -webkit-transition: all 0.3s; + -moz-transition: all 0.3s; + transition: all 0.3s; +} + +ul.social-buttons li a:hover, +ul.social-buttons li a:focus, +ul.social-buttons li a:active { + background-color: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +.btn:focus, +.btn:active, +.btn.active, +.btn:active:focus { + outline: 0; +} + +.portfolio-modal .modal-content { + padding: 100px 0; + min-height: 100%; + border: 0; + border-radius: 0; + text-align: center; + background-clip: border-box; + -webkit-box-shadow: none; + box-shadow: none; +} + +.portfolio-modal .modal-content h2 { + margin-bottom: 15px; + font-size: 3em; +} + +.portfolio-modal .modal-content p { + margin-bottom: 30px; +} + +.portfolio-modal .modal-content p.item-intro { + margin: 20px 0 30px; + font-family: HelveticaNeue-Thin, Helvetica, Arial, sans-serif; + font-size: 16px; + font-style: italic; +} + +.portfolio-modal .modal-content ul.list-inline { + margin-top: 0; + margin-bottom: 30px; +} + +.portfolio-modal .modal-content img { + margin-bottom: 30px; +} + +.portfolio-modal .close-modal { + position: absolute; + top: 25px; + right: 25px; + width: 75px; + height: 75px; + background-color: transparent; + cursor: pointer; +} + +.portfolio-modal .close-modal:hover { + opacity: 0.3; +} + +.portfolio-modal .close-modal .lr { + z-index: 1051; + width: 1px; + height: 75px; + margin-left: 35px; + background-color: #222; + -webkit-transform: rotate(45deg); + -ms-transform: rotate(45deg); + transform: rotate(45deg); +} + +.portfolio-modal .close-modal .lr .rl { + z-index: 1052; + width: 1px; + height: 75px; + background-color: #222; + -webkit-transform: rotate(90deg); + -ms-transform: rotate(90deg); + transform: rotate(90deg); +} + +::-moz-selection { + text-shadow: none; + background: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +::selection { + text-shadow: none; + background: HelveticaNeue, Helvetica, Arial, sans-serif; +} + +img::selection { + background: 0 0; +} + +img::-moz-selection { + background: 0 0; +} + +body { + webkit-tap-highlight-color: HelveticaNeue, Helvetica, Arial, sans-serif; +} diff --git a/client/js/app/src/app/styles/vespa.css b/client/js/app/src/app/styles/vespa.css new file mode 100644 index 00000000000..4017f0275c3 --- /dev/null +++ b/client/js/app/src/app/styles/vespa.css @@ -0,0 +1,752 @@ +/** +* Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +* Based on vespa.css from https://github.com/vespa-engine/frontpage for header- and footer-elements +*/ +:root { + --primary: #188fff; + --secondary: #003abc; + --secondary-dark: #333; + --muted: #777; + + --fontprimary: 'Hind Madurai', Helvetica, Arial, sans-serif; + --fontsecondary: 'Hind Madurai', Helvetica, Arial, sans-serif; +} + +.bg-light-blue { + background-image: linear-gradient( + -1deg, + rgba(63, 157, 216, 0.08) 0%, + rgba(163, 195, 215, 0.08) 97% + ); +} + +.yql { +} + +.yql:focus { + outline: none; +} + +.information { + margin-left: -40px; + margin-right: 28px; + margin-top: -2.5px; + opacity: 0.6; + transition: transform 0.1s; /* Animation */ +} + +.information:hover { + transform: scale(1.05); +} + +/*** Tooltips! ***/ +.tip { + visibility: visible; + border-bottom: 1px dotted; + position: relative; + cursor: help; + text-decoration: none; + color: #fff !important; +} +.tip span { + display: none; + z-index: 100; + position: absolute; + padding: 0.6em; + padding-left: 1em; + top: 1.5em; + left: 2.4em; + width: 15em; + background-color: #4da2d6; + border: 1px solid #ffffff; + border-radius: 0.5em; +} + +.tip span td { + text-align: left; + vertical-align: top; + padding-left: 5px; +} + +.tip span a { + text-decoration: none; + color: white; +} + +.tip:hover span { + display: inline-block; +} +.sg-question-set, +.sg-type-radio ul.sg-list-vertical li, +.sg-type-checkbox ul.sg-list-vertical li, +.sg-question-options, +.sg-type-radio-likert .sg-question-options, +.sg-type-table .sg-question-options, +.sg-instructions { + overflow: visible; +} + +section h2.section-heading, +section h2.section-subheading, +section h3.section-heading, +section h3.section-subheading { + text-transform: none; + font-family: 'Hind Madurai', Helvetica, Arial, sans-serif; + font-weight: normal; + font-style: normal; +} + +section h2.section-heading, +section h3.section-heading { + font-size: 50px; + color: #3f9dd8; + margin-bottom: 5px; +} + +section h2.section-subheading, +section h3.section-subheading { + line-height: 26px; + margin-top: 0; +} + +header .help-title { + color: #ffc43c; + text-transform: uppercase; + font-weight: bold; +} + +.text-muted { + color: #303030; +} + +#banner { + background-color: #336699; + color: #ffffff; + font-weight: bold; + padding-bottom: 2px; + padding-top: 2px; + width: 100%; +} + +#banner a { + color: #ffffff; + text-decoration: underline; +} + +.navbar-default { + background-color: #005a8e; + padding: 0; + border-bottom: 2px solid rgba(255, 255, 255, 0.2); +} + +.navbar-default .navbar-header { + margin-left: 13px; +} + +.navbar-default .navbar-brand { + background: transparent url('https://vespa.ai/assets/vespa-logo.png') + no-repeat; + background-size: contain; + direction: ltr; + text-indent: -9000px; + height: 28px; + width: 100px; + display: inline-block; + font-family: 'Hind Madurai', Helvetica, Arial, sans-serif; + font-weight: bold; + color: var(--primary); + margin-top: 16px; + margin-left: 10px; +} + +.navbar-nav .nav-link { + font-weight: bold; + font-size: 14px; + letter-spacing: 1px; + color: #ffffff; + text-transform: none; + font-family: 'Hind Madurai', Helvetica, Arial, sans-serif; +} + +.navbar-default .navbar-nav > a:hover, +.navbar-default .navbar-nav > .active > a, +.navbar-default .navbar-nav > .active > a:hover { + color: #ffc43c; +} + +.navbar-default .navbar-toggler-icon { + background-image: url('data:image/svg+xml,%3csvg xmlns=%27http://www.w3.org/2000/svg%27 viewBox=%270 0 30 30%27%3e%3cpath stroke=%27rgba%280, 0, 0, 0.55%29%27 stroke-linecap=%27round%27 stroke-miterlimit=%2710%27 stroke-width=%272%27 d=%27M4 7h22M4 15h22M4 23h22%27/%3e%3c/svg%3e'); +} + +.navbar-default .navbar-toggler { + margin-right: 45px; + border: 1px solid; + border-color: #1a242f; +} +.navbar-toggler:hover, +.navbar-default .navbar-toggler:focus { + background-color: #1a242f; + box-shadow: 0 0 0 0; +} + +.navbar-toggle { + margin-right: 45px; +} + +/* Trick to prevent anchored links to hide heading behind navbar */ +section[id]:before { + display: block; + content: ' '; + margin-top: -80px; + height: 100px; + visibility: hidden; +} + +header { + /*background-image: linear-gradient(0deg, #98C1DB 7%, #3F9DD8 100%); */ + background-color: #005a8e; + min-height: 1150px; +} + +@media (max-height: 1150px) { + header { + min-height: 100vh; + } +} + +header .intro { + margin-top: -80px; + max-width: 350px; + padding-top: 10px; + padding-bottom: 50px; +} + +header .intro-logo { + max-width: 120px; + padding-bottom: 30px; +} + +header .intro-button { + box-shadow: 0px 2px 4px 0px rgba(0, 0, 0, 0.5); + padding-left: 50px; + padding-right: 50px; +} + +header .intro-long { + line-height: 27px; + font-size: 16px; + padding-bottom: 20px; +} + +header .intro-param { + line-height: 27px; + font-size: 16px; + padding-bottom: 0px; + margin-bottom: 5px; +} +header .response { + line-height: 27px; + font-size: 20px; + padding-top: 10px; + padding-bottom: 8px; +} + +header .methodselector { + font-size: 15px; + color: #fff; + background-color: #99c1da; + width: 70px; + height: 25px; + border-width: 0px; + -o-transition: 0.3s; + -ms-transition: 0.3s; + -moz-transition: 0.3s; + -webkit-transition: 0.3s; + transition: 0.3s; +} +header .intro-help:hover { + background-color: #4ea2d6 !important; +} +header .methodselector:hover { + background-color: #79b4d8; +} +header .button:hover { + background-color: #79b4d8; +} +header .removeRow:hover { + background-color: #79b4d8; +} +header .addRow:hover { + background-color: #79b4d8; +} +header .showJSON:hover { + background-color: #79b4d8; +} +header .copyURL:hover { + background-color: #79b4d8; +} +header .copyJSON:hover { + background-color: #79b4d8; +} +header .methodselector:focus { + outline: none; +} +header .textbox:focus { + outline: none; +} +header .input:focus { + outline: none; +} +header .responsebox::selection { + background-color: grey; +} +header .textbox::selection { + background-color: grey; +} +header .input::selection { + background-color: grey; +} +header .propvalue::selection { + background-color: grey; +} +header .responsebox:focus { + outline: none; +} +header .responseinfo:focus { + outline: none; +} +header .propvalue:focus { + outline: none; +} +header .addpropsbutton:focus { + outline: none; +} + +header .input { + color: #000; + width: 190px; + border-width: 0px; + margin-right: 2px; +} + +header .input2 { + color: #000; + width: 175px; + border-width: 0px; + margin-right: 2px; +} + +header .textbox { + color: #000; + padding-left: 2px; +} + +header .responsebox { + color: #000; + background-color: #fff; +} + +header .responseinfo { + color: #d6e6f0; + text-align: center; + border-width: 0px; + background-color: transparent; +} + +header .propvalue { + color: #000; + background-color: #fff; + width: 175px; + border-width: 0px; + margin-bottom: 3px; +} + +header .propvalue::-webkit-input-placeholder { + /* Safari, Chrome(, Opera?) */ + color: gray; + font-style: italic; +} +header .propvalue:-moz-placeholder { + /* Firefox 18- */ + color: gray; + font-style: italic; +} +header .propvalue:-moz-placeholder { + /* Firefox 19+ */ + color: gray; + font-style: italic; +} +header .propvalue-ms-input-placeholder { + /* IE (10+?) */ + color: gray; + font-style: italic; +} + +header .button { + font-size: 15px; + color: #fff; + background-color: #99c1da; + width: 70px; + height: 25px; + border-width: 0px; + border-radius: 5px; + padding: 0px; +} +header .button:focus { + outline: none; +} + +header .addpropsbutton { + font-size: 15px; + color: #fff; + background-color: #99c1da; + width: 175px; + height: 23px; + border-width: 0px; + border-radius: 5px; + padding: 0px; + margin-left: 2px; + margin-bottom: 3px; +} +header .addRow { + font-size: 15px; + color: #fff; + background-color: #99c1da; + width: 70px; + height: 25px; + border-width: 0px; + border-radius: 5px; + padding: 0px; +} +header .addRow:focus { + outline: none; +} + +header .removeRow { + font-size: 16px; + color: #fff; + background-color: #99c1da; + width: 35px; + height: 23px; + border-width: 0px; + border-radius: 5px; + padding: 0px; + padding-bottom: 2px; + margin-left: 3px; + text-align: center; + line-height: normal; +} +header .removeRow:focus { + outline: none; +} + +header .json:focus { + outline: none; +} +header .copyJSON:focus { + outline: none; +} +header .showJSON:focus { + outline: none; +} +header .copyURL:focus { + outline: none; +} +header .pasteJSON:focus { + outline: none; +} + +header .copyJSON { + font-size: 15px; + color: #fff; + background-color: #99c1da; + width: 130px; + height: 25px; + border-width: 0px; + border-radius: 5px; + padding-left: 1px; + margin-top: 10px; + margin-bottom: 20px; + margin-right: 2px; + display: none; +} + +header .showJSON { + font-size: 15px; + color: #fff; + background-color: #99c1da; + width: 140px; + height: 25px; + border-width: 0px; + border-radius: 5px; + padding: 0px; + margin-top: 5px; + margin-bottom: 10px; +} + +header .pasteJSON { + font-size: 15px; + color: #fff; + background-color: #99c1da; + width: 140px; + height: 25px; + border-width: 0px; + border-radius: 5px; + padding-left: 0px; + padding-bottom: 2px; + margin-top: 35px; + margin-bottom: 10px; +} + +header .copyURL { + font-size: 15px; + color: #fff; + background-color: #99c1da; + width: 130px; + height: 25px; + border-width: 0px; + border-radius: 5px; + padding-left: 1px; + margin-top: 25px; + margin-bottom: 10px; + margin-left: 2px; + display: none; +} + +header .intro .intro-lead-in { + font-style: normal; + font-size: 22px; + line-height: 50px; + margin-bottom: 25px; + font-family: 'Hind Madurai', Helvetica, Arial, sans-serif; +} + +header .intro .intro-heading { + font-style: normal; +} + +header .intro-button { + box-shadow: 0px 2px 4px 0px rgba(0, 0, 0, 0.5); +} + +header .intro .btn-xl { + background: #ffffff; + border: 1px solid #3f9dd8; + border-radius: 2px; + font-size: 20px; + color: #3f9dd8; + font-weight: normal; + text-transform: none; + font-family: 'Hind Madurai', Helvetica, Arial, sans-serif; +} + +header span { + display: inline-block; + position: inherit; + margin-top: 10px; + /*padding-left: 50px;*/ + transform: translateX(-50%); + background: transparent; + border: none !important; + font-size: 0; +} + +header .intro-copy { + background: transparent; + border: none !important; + font-size: 0; +} + +header .intro-help { + display: inline-block; + margin-top: 15px; + width: 134px; + height: 45px; + background-color: transparent; + border: none !important; + font-size: 0; + + border-top-left-radius: 30%; + border-top-right-radius: 29%; + border-bottom-right-radius: 24%; + border-bottom-left-radius: 26%; +} + +header .intro-refresh { + display: inline-block; + margin-top: 15px; + /* transform: translateX(-50%); */ + background: transparent; + border: none !important; + font-size: 0; +} + +@media (min-height: 1150px) { + header .intro-down-arrow { + /* Hard code top: 1150 (height of top section) - 20 (bottom) - 36 (image height) = 1094 */ + top: 1094px; + } +} + +@media (max-height: 825px) { + header .intro-down-arrow { + display: none; + } +} + +@media (min-width: 768px) { + header .intro { + padding-top: 200px; + padding-bottom: 200px; + max-width: 620px; + } + + header .intro-logo { + max-width: 238px; + padding-bottom: 70px; + } + + header .intro .intro-lead-in { + font-size: 51px; + font-style: normal; + line-height: 50px; + } + + header .intro .intro-long { + margin-left: 80px; + margin-right: 80px; + } + + header .intro .intro-heading { + margin-bottom: 25px; + font-size: 90px; + font-weight: 600; + } +} + +section { + padding: 50px 0; +} + +section h3.section-subheading { + margin-bottom: 50px; + font-weight: normal; +} + +@media (min-width: 768px) { + section { + padding: 50px 0; + } +} + +footer .row { + font-size: 12px; + text-align: left; +} + +footer .row h4 { + font-size: 14px; + padding-left: 40px; +} + +footer .row .quicklinks { + line-height: 25px; + list-style: none; + text-align: left; +} + +footer .credits { + font-size: 10px; + text-align: center; +} + +#description p { + font-weight: 300; + font-size: 25px; +} + +footer { + font-size: 14px; + color: #ffffff; + background-color: #3f9dd8; + line-height: 22px; + letter-spacing: 0; + text-align: left; + display: flex; + flex-wrap: wrap; +} + +footer .footer-row { + display: -webkit-box; + display: -webkit-flex; + display: -ms-flexbox; + display: flex; + flex-wrap: wrap; +} + +footer quicklinks { + font-size: 14px; + color: #ffffff; +} + +footer ul.quicklinks { + margin: 0; + padding: 0; +} + +footer .footer-title { + color: #ffc43c; + text-transform: uppercase; + font-weight: bold; +} + +footer ul.quicklinks a { + font-size: 14px; + color: #ffffff; +} + +footer .quicklink-section { + padding: 0; + min-width: 120px; +} + +footer .credits { + float: left; + font-size: 12px; + font-weight: normal; + text-align: right; + font-family: 'Hind Madurai', Helvetica, Arial, sans-serif; +} + +footer .credits a { + color: #ffffff; +} + +footer .credits span { + color: #ffc43c; +} + +footer a { + text-decoration: none; +} + +footer .table { + width: fit-content; + margin-right: auto; + margin-left: auto; + float: left; + margin-bottom: 0em; +} + +footer .table tbody th { + font-size: 13px; + padding-right: 2em; +} + +footer table thead { + color: #ffc43c; +} + +footer table tbody a { + color: #ffffff; +} diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index f1cee14e50e..2261621f31b 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -208,6 +208,13 @@ dependencies: regenerator-runtime "^0.13.4" +"@babel/runtime@^7.13.16", "@babel/runtime@^7.17.2", "@babel/runtime@^7.5.5", "@babel/runtime@^7.6.2", "@babel/runtime@^7.6.3", "@babel/runtime@^7.8.7": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.18.6.tgz#6a1ef59f838debd670421f8c7f2cbb8da9751580" + integrity sha512-t9wi7/AW6XtKahAe20Yw0/mMljKq0B1r2fPdvaAdV/KPDZewFXdaaa6K7lxmZBZ8FBNpCiAT6iHPmd6QO9bKfQ== + dependencies: + regenerator-runtime "^0.13.4" + "@babel/template@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.16.7.tgz#8d126c8701fde4d66b264b3eba3d96f07666d155" @@ -463,7 +470,7 @@ clsx "^1.1.1" csstype "3.0.9" -"@popperjs/core@^2.9.3": +"@popperjs/core@^2.10.1", "@popperjs/core@^2.9.3": version "2.11.5" resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.5.tgz#db5a11bf66bdab39569719555b0f76e138d7bd64" integrity sha512-9X2obfABZuDVLCgPK9aX0a/x4jaOEweTTWE2+9sr0Qqqevj2Uv5XorvusThmc9XGYpS9yI+fhh8RTafBtGposw== @@ -558,6 +565,35 @@ dependencies: "@babel/runtime" "^7.13.10" +"@react-aria/ssr@^3.0.1": + version "3.2.0" + resolved "https://registry.yarnpkg.com/@react-aria/ssr/-/ssr-3.2.0.tgz#88460384b43204f91c972d5b0de24ee44d6a2984" + integrity sha512-wwJFdkl+Q8NU5yJ4NvdAOqx5LM3QtUVoSjuK7Ey8jZ4WS4bB0EqT3Kr3IInBs257HzZ5nXCiKXKE4NGXXuIRWA== + dependencies: + "@babel/runtime" "^7.6.2" + +"@restart/hooks@^0.4.0", "@restart/hooks@^0.4.6": + version "0.4.7" + resolved "https://registry.yarnpkg.com/@restart/hooks/-/hooks-0.4.7.tgz#d79ca6472c01ce04389fc73d4a79af1b5e33cd39" + integrity sha512-ZbjlEHcG+FQtpDPHd7i4FzNNvJf2enAwZfJbpM8CW7BhmOAbsHpZe3tsHwfQUrBuyrxWqPYp2x5UMnilWcY22A== + dependencies: + dequal "^2.0.2" + +"@restart/ui@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@restart/ui/-/ui-1.2.0.tgz#fb90251aa25f99b41ccedc78a91d2a15f3c5e0fb" + integrity sha512-oIh2t3tG8drZtZ9SlaV5CY6wGsUViHk8ZajjhcI+74IQHyWy+AnxDv8rJR5wVgsgcgrPBUvGNkC1AEdcGNPaLQ== + dependencies: + "@babel/runtime" "^7.13.16" + "@popperjs/core" "^2.10.1" + "@react-aria/ssr" "^3.0.1" + "@restart/hooks" "^0.4.0" + "@types/warning" "^3.0.0" + dequal "^2.0.2" + dom-helpers "^5.2.0" + uncontrollable "^7.2.1" + warning "^4.0.3" + "@rollup/pluginutils@^4.2.1": version "4.2.1" resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-4.2.1.tgz#e6c6c3aba0744edce3fb2074922d3776c0af2a6d" @@ -588,6 +624,13 @@ dependencies: "@types/react" "*" +"@types/react-transition-group@^4.4.4": + version "4.4.4" + resolved "https://registry.yarnpkg.com/@types/react-transition-group/-/react-transition-group-4.4.4.tgz#acd4cceaa2be6b757db61ed7b432e103242d163e" + integrity sha512-7gAPz7anVK5xzbeQW9wFBDg7G++aPLAFY0QaSMOou9rJZpbuI58WAuJrgu+qR92l61grlnCUe7AFX8KGahAgug== + dependencies: + "@types/react" "*" + "@types/react@*": version "18.0.13" resolved "https://registry.yarnpkg.com/@types/react/-/react-18.0.13.tgz#0f5bd24a5f26593e04e450fe85ff43f51c1524ff" @@ -597,7 +640,7 @@ "@types/scheduler" "*" csstype "^3.0.2" -"@types/react@^18": +"@types/react@>=16.9.11", "@types/react@^18": version "18.0.14" resolved "https://registry.yarnpkg.com/@types/react/-/react-18.0.14.tgz#e016616ffff51dba01b04945610fe3671fdbe06d" integrity sha512-x4gGuASSiWmo0xjDLpm5mPb52syZHJx02VKbqUKdLmKtAwIh63XClGsiTI1K6DO5q7ox4xAsQrU+Gl3+gGXF9Q== @@ -611,6 +654,11 @@ resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39" integrity sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew== +"@types/warning@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@types/warning/-/warning-3.0.0.tgz#0d2501268ad8f9962b740d387c4654f5f8e23e52" + integrity sha512-t/Tvs5qR47OLOr+4E9ckN8AmP2Tf16gWq+/qA4iUGS/OOyHVO8wv2vjJuX8SNOUTJyWb+2t7wJm6cXILFnOROA== + "@vitejs/plugin-react@^1": version "1.3.2" resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-1.3.2.tgz#2fcf0b6ce9bcdcd4cec5c760c199779d5657ece1" @@ -782,6 +830,11 @@ chalk@^4.0.0: ansi-styles "^4.1.0" supports-color "^7.1.0" +classnames@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.3.1.tgz#dfcfa3891e306ec1dad105d0e88f4417b8535e8e" + integrity sha512-OlQdbZ7gLfGarSqxesMesDa5uz7KFbID8Kpq/SxIoNGDqY8lSYs0D+hhtBXhcdB3rcbXArFr7vlHheLk1voeNA== + clsx@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.1.1.tgz#98b3134f9abbdf23b2663491ace13c5c03a73188" @@ -876,6 +929,11 @@ define-properties@^1.1.3, define-properties@^1.1.4: has-property-descriptors "^1.0.0" object-keys "^1.1.1" +dequal@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/dequal/-/dequal-2.0.2.tgz#85ca22025e3a87e65ef75a7a437b35284a7e319d" + integrity sha512-q9K8BlJVxK7hQYqa6XISGmBZbtQQWVXSrRrWreHC94rMt1QL/Impruc+7p2CYSYuVIUr+YCt6hjrs1kkdJRTug== + doctrine@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.1.0.tgz#5cd01fc101621b42c4cd7f5d1a66243716d3f39d" @@ -890,6 +948,14 @@ doctrine@^3.0.0: dependencies: esutils "^2.0.2" +dom-helpers@^5.0.1, dom-helpers@^5.2.0, dom-helpers@^5.2.1: + version "5.2.1" + resolved "https://registry.yarnpkg.com/dom-helpers/-/dom-helpers-5.2.1.tgz#d9400536b2bf8225ad98fe052e029451ac40e902" + integrity sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA== + dependencies: + "@babel/runtime" "^7.8.7" + csstype "^3.0.2" + electron-to-chromium@^1.4.147: version "1.4.158" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.158.tgz#abbdaaf64676bfa4bc0307522125db34424a0ada" @@ -1546,6 +1612,13 @@ internal-slot@^1.0.3: has "^1.0.3" side-channel "^1.0.4" +invariant@^2.2.4: + version "2.2.4" + resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6" + integrity sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA== + dependencies: + loose-envify "^1.0.0" + is-bigint@^1.0.1: version "1.0.4" resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.4.tgz#08147a1875bc2b32005d41ccd8291dffc6691df3" @@ -2011,7 +2084,15 @@ pretty-quick@^3: mri "^1.1.5" multimatch "^4.0.0" -prop-types@^15.8.1: +prop-types-extra@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/prop-types-extra/-/prop-types-extra-1.1.1.tgz#58c3b74cbfbb95d304625975aa2f0848329a010b" + integrity sha512-59+AHNnHYCdiC+vMwY52WmvP5dM3QLeoumYuEyceQDi9aEhtwN9zIQ2ZNo25sMyXnbh32h+P1ezDsUpUH3JAew== + dependencies: + react-is "^16.3.2" + warning "^4.0.0" + +prop-types@^15.6.2, prop-types@^15.8.1: version "15.8.1" resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5" integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg== @@ -2033,6 +2114,24 @@ punycode@^2.1.0: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== +react-bootstrap@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/react-bootstrap/-/react-bootstrap-2.4.0.tgz#99bf9656e2e7a23ae1ae135d18fd5ad7c344b416" + integrity sha512-dn599jNK1Fg5GGjJH+lQQDwELVzigh/MdusKpB/0el+sCjsO5MZDH5gRMmBjRhC+vb7VlCDr6OXffPIDSkNMLw== + dependencies: + "@babel/runtime" "^7.17.2" + "@restart/hooks" "^0.4.6" + "@restart/ui" "^1.2.0" + "@types/react-transition-group" "^4.4.4" + classnames "^2.3.1" + dom-helpers "^5.2.1" + invariant "^2.2.4" + prop-types "^15.8.1" + prop-types-extra "^1.1.0" + react-transition-group "^4.4.2" + uncontrollable "^7.2.1" + warning "^4.0.3" + react-dom@^18: version "18.2.0" resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-18.2.0.tgz#22aaf38708db2674ed9ada224ca4aa708d821e3d" @@ -2046,11 +2145,16 @@ react-fast-compare@^3.0.1: resolved "https://registry.yarnpkg.com/react-fast-compare/-/react-fast-compare-3.2.0.tgz#641a9da81b6a6320f270e89724fb45a0b39e43bb" integrity sha512-rtGImPZ0YyLrscKI9xTpV8psd6I8VAtjKCzQDlzyDvqJA8XOW78TXYQwNRNd8g8JZnDu8q9Fu/1v4HPAVwVdHA== -react-is@^16.13.1, react-is@^16.7.0: +react-is@^16.13.1, react-is@^16.3.2, react-is@^16.7.0: version "16.13.1" resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4" integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ== +react-lifecycles-compat@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz#4f1a273afdfc8f3488a8c516bfda78f872352362" + integrity sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA== + react-popper@^2.2.5: version "2.3.0" resolved "https://registry.yarnpkg.com/react-popper/-/react-popper-2.3.0.tgz#17891c620e1320dce318bad9fede46a5f71c70ba" @@ -2064,7 +2168,7 @@ react-refresh@^0.13.0: resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.13.0.tgz#cbd01a4482a177a5da8d44c9755ebb1f26d5a1c1" integrity sha512-XP8A9BT0CpRBD+NYLLeIhld/RqG9+gktUjW1FkE+Vm7OCinbG1SshcK5tb9ls4kzvjZr9mOQc7HYgBngEyPAXg== -react-router-dom@^6: +react-router-dom@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.3.0.tgz#a0216da813454e521905b5fa55e0e5176123f43d" integrity sha512-uaJj7LKytRxZNQV8+RbzJWnJ8K2nPsOOEuX7aQstlMZKQT0164C+X2w6bnkqU3sjtLvpd5ojrezAyfZ1+0sStw== @@ -2088,6 +2192,16 @@ react-textarea-autosize@^8.3.2: use-composed-ref "^1.3.0" use-latest "^1.2.1" +react-transition-group@^4.4.2: + version "4.4.2" + resolved "https://registry.yarnpkg.com/react-transition-group/-/react-transition-group-4.4.2.tgz#8b59a56f09ced7b55cbd53c36768b922890d5470" + integrity sha512-/RNYfRAMlZwDSr6z4zNKV6xu53/e2BuaBbGhbyYIXTrmgu/bGHzmqOs7mJSJBHy9Ud+ApHx3QjrkKSp1pxvlFg== + dependencies: + "@babel/runtime" "^7.5.5" + dom-helpers "^5.0.1" + loose-envify "^1.4.0" + prop-types "^15.6.2" + react@^18: version "18.2.0" resolved "https://registry.yarnpkg.com/react/-/react-18.2.0.tgz#555bd98592883255fa00de14f1151a917b5d77d5" @@ -2328,6 +2442,16 @@ unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" +uncontrollable@^7.2.1: + version "7.2.1" + resolved "https://registry.yarnpkg.com/uncontrollable/-/uncontrollable-7.2.1.tgz#1fa70ba0c57a14d5f78905d533cf63916dc75738" + integrity sha512-svtcfoTADIB0nT9nltgjujTi7BzVmwjZClOmskKu/E8FW9BXzg9os8OLr4f8Dlnk0rYWJIWr4wv9eKUXiQvQwQ== + dependencies: + "@babel/runtime" "^7.6.3" + "@types/react" ">=16.9.11" + invariant "^2.2.4" + react-lifecycles-compat "^3.0.4" + uri-js@^4.2.2: version "4.4.1" resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" @@ -2358,9 +2482,9 @@ v8-compile-cache@^2.0.3: integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== vite@^2: - version "2.9.12" - resolved "https://registry.yarnpkg.com/vite/-/vite-2.9.12.tgz#b1d636b0a8ac636afe9d83e3792d4895509a941b" - integrity sha512-suxC36dQo9Rq1qMB2qiRorNJtJAdxguu5TMvBHOc/F370KvqAe9t48vYp+/TbPKRNrMh/J55tOUmkuIqstZaew== + version "2.9.13" + resolved "https://registry.yarnpkg.com/vite/-/vite-2.9.13.tgz#859cb5d4c316c0d8c6ec9866045c0f7858ca6abc" + integrity sha512-AsOBAaT0AD7Mhe8DuK+/kE4aWYFMx/i0ZNi98hJclxb4e0OhQcZYUrvLjIaQ8e59Ui7txcvKMiJC1yftqpQoDw== dependencies: esbuild "^0.14.27" postcss "^8.4.13" @@ -2369,7 +2493,7 @@ vite@^2: optionalDependencies: fsevents "~2.3.2" -warning@^4.0.2: +warning@^4.0.0, warning@^4.0.2, warning@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/warning/-/warning-4.0.3.tgz#16e9e077eb8a86d6af7d64aa1e05fd85b4678ca3" integrity sha512-rpJyN222KWIvHJ/F53XSZv0Zl/accqHR8et1kpaMTD/fLCRxtV8iX8czMzY7sVZupTI3zcUTg8eycS2kNF9l6w== diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 5e9f1f52a0c..4536257f8a3 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -77,13 +77,19 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { return "THROUGHPUT"; } @ModelFeatureFlag(owners = {"baldersheim"}) default String responseSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultNumResponseThreads() { return 2; } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipCommunicationManagerThread() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusRequestThread() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusReplyThread() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean mbusDispatchOnDecode() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean mbusDispatchOnEncode() { return true; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipCommunicationManagerThread() { return true; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipMbusRequestThread() { return true; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipMbusReplyThread() { return true; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="8.15") default boolean mbusDispatchOnDecode() { return true; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="8.15") default boolean mbusDispatchOnEncode() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default int mbusNetworkThreads() { return 1; } - @ModelFeatureFlag(owners = {"baldersheim"}) default int mbusThreads() { return 4; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="8.15") default int mbusThreads() { return 4; } + @ModelFeatureFlag(owners = {"baldersheim"}) default int mbusJavaRpcNumTargets() { return 1; } + @ModelFeatureFlag(owners = {"baldersheim"}) default int mbusJavaEventsBeforeWakeup() { return 1; } + @ModelFeatureFlag(owners = {"baldersheim"}) default int mbusCppRpcNumTargets() { return 1; } + @ModelFeatureFlag(owners = {"baldersheim"}) default int mbusCppEventsBeforeWakeup() { return 1; } + @ModelFeatureFlag(owners = {"baldersheim"}) default int rpcNumTargets() { return 1; } + @ModelFeatureFlag(owners = {"baldersheim"}) default int rpcEventsBeforeWakeup() { return 1; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedNiceness() { return 0.0; } @@ -108,7 +114,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List<String> ignoredHttpUserAgents() { return List.of(); } - @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } + @ModelFeatureFlag(owners = {"bjorncs"}, removeAfter="7.last") default boolean enableServerOcspStapling() { return true; } @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsDecrementFactor() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsBackoff() { throw new UnsupportedOperationException("TODO specify default value"); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 5f189a63701..400aea86834 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -78,9 +78,12 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean enableBitVectors = false; private boolean loadCodeAsHugePages = false; private boolean sharedStringRepoNoReclaim = false; - private boolean mbus_dispatch_on_decode = true; - private boolean mbus_dispatch_on_encode = true; - private int mbus_threads = 4; + private int mbus_java_num_targets = 1; + private int mbus_java_events_before_wakeup = 1; + private int mbus_cpp_num_targets = 1; + private int mbus_cpp_events_before_wakeup = 1; + private int rpc_num_targets = 1; + private int rpc_events_before_wakeup = 1; private int mbus_network_threads = 1; private Architecture adminClusterNodeResourcesArchitecture = Architecture.getDefault(); @@ -137,10 +140,13 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Architecture adminClusterArchitecture() { return adminClusterNodeResourcesArchitecture; } @Override public boolean sharedStringRepoNoReclaim() { return sharedStringRepoNoReclaim; } @Override public boolean loadCodeAsHugePages() { return loadCodeAsHugePages; } - @Override public boolean mbusDispatchOnDecode() { return mbus_dispatch_on_decode; } - @Override public boolean mbusDispatchOnEncode() { return mbus_dispatch_on_encode; } @Override public int mbusNetworkThreads() { return mbus_network_threads; } - @Override public int mbusThreads() { return mbus_threads; } + @Override public int mbusJavaRpcNumTargets() { return mbus_java_num_targets; } + @Override public int mbusJavaEventsBeforeWakeup() { return mbus_java_events_before_wakeup; } + @Override public int mbusCppRpcNumTargets() { return mbus_cpp_num_targets; } + @Override public int mbusCppEventsBeforeWakeup() { return mbus_cpp_events_before_wakeup; } + @Override public int rpcNumTargets() { return rpc_num_targets; } + @Override public int rpcEventsBeforeWakeup() { return rpc_events_before_wakeup; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { @@ -367,24 +373,35 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setMbusDispatchOnDecode(boolean value) { - this.mbus_dispatch_on_decode = value; + public TestProperties setMbusNetworkThreads(int value) { + this.mbus_network_threads = value; return this; } - public TestProperties setMbusDispatchOnEncode(boolean value) { - this.mbus_dispatch_on_encode = value; + public TestProperties setMbusJavaRpcNumTargets(int value) { + this.mbus_java_num_targets = value; return this; } - - public TestProperties setMbusThreads(int value) { - this.mbus_threads = value; + public TestProperties setMbusJavaEventsBeforeWakeup(int value) { + this.mbus_java_events_before_wakeup = value; return this; } - - public TestProperties setMbusNetworkThreads(int value) { - this.mbus_network_threads = value; + public TestProperties setMbusCppEventsBeforeWakeup(int value) { + this.mbus_cpp_events_before_wakeup = value; return this; } + public TestProperties setMbusCppRpcNumTargets(int value) { + this.mbus_cpp_num_targets = value; + return this; + } + public TestProperties setRpcNumTargets(int value) { + this.rpc_num_targets = value; + return this; + } + public TestProperties setRpcEventsBeforeWakeup(int value) { + this.rpc_events_before_wakeup = value; + return this; + } + public TestProperties setAdminClusterNodeResourcesArchitecture(Architecture architecture) { this.adminClusterNodeResourcesArchitecture = architecture; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 15a0e060e1c..33c125dcecf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -7,10 +7,12 @@ import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerThreadpool; +import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.UserBindingPattern; +import java.nio.file.Path; import java.util.Collection; import java.util.Collections; @@ -21,6 +23,8 @@ import java.util.Collections; public class ContainerDocumentApi { public static final String DOCUMENT_V1_PREFIX = "/document/v1"; + public static final Path VESPACLIENT_CONTAINER_BUNDLE = + PlatformBundles.absoluteBundlePath("vespaclient-container-plugin"); private final boolean ignoreUndefinedFields; @@ -28,6 +32,11 @@ public class ContainerDocumentApi { this.ignoreUndefinedFields = ignoreUndefinedFields; addRestApiHandler(cluster, handlerOptions); addFeedHandler(cluster, handlerOptions); + addVespaClientContainerBundle(cluster); + } + + public static void addVespaClientContainerBundle(ContainerCluster<?> c) { + c.addPlatformBundle(VESPACLIENT_CONTAINER_BUNDLE); } private static void addFeedHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 47dab37cc14..9997b20d205 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -27,7 +27,6 @@ public final class ApplicationContainer extends Container implements private static final String defaultHostedJVMArgs = "-XX:+SuppressFatalErrorMessage"; private final boolean isHostedVespa; - private final boolean enableServerOcspStapling; public ApplicationContainer(AbstractConfigProducer<?> parent, String name, int index, DeployState deployState) { this(parent, name, false, index, deployState); @@ -36,7 +35,6 @@ public final class ApplicationContainer extends Container implements public ApplicationContainer(AbstractConfigProducer<?> parent, String name, boolean retired, int index, DeployState deployState) { super(parent, name, retired, index, deployState); this.isHostedVespa = deployState.isHosted(); - this.enableServerOcspStapling = deployState.featureFlags().enableServerOcspStapling(); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder")); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); @@ -68,12 +66,10 @@ public final class ApplicationContainer extends Container implements if (hasDocproc()) { b.append(ApplicationContainer.defaultHostedJVMArgs).append(' '); } - if (enableServerOcspStapling) { - b.append("-Djdk.tls.server.enableStatusRequestExtension=true ") - .append("-Djdk.tls.stapling.responseTimeout=2000 ") - .append("-Djdk.tls.stapling.cacheSize=256 ") - .append("-Djdk.tls.stapling.cacheLifetime=3600 "); - } + b.append("-Djdk.tls.server.enableStatusRequestExtension=true ") + .append("-Djdk.tls.stapling.responseTimeout=2000 ") + .append("-Djdk.tls.stapling.cacheSize=256 ") + .append("-Djdk.tls.stapling.cacheLifetime=3600 "); } String jvmArgs = super.getJvmOptions(); if (!jvmArgs.isBlank()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index cb4fe8f67ca..937d7cf58d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -78,7 +78,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public static final int heapSizePercentageOfTotalNodeMemory = 70; public static final int heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster = 18; - private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); private final Set<String> previousHosts; @@ -89,6 +88,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private MbusParams mbusParams; private boolean messageBusEnabled = true; + private final int transport_events_before_wakeup; + private final int transport_connections_per_target; private Integer memoryPercentage = null; @@ -114,14 +115,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addMetricsHandlers(); addTestrunnerComponentsIfTester(deployState); - addPlatformBundlesForApplicationCluster(); - } - - private void addPlatformBundlesForApplicationCluster() { - Set<String> bundles = Set.of( - "container-search-and-docproc", "container-search-gui", "docprocs", - "linguistics-components", "vespaclient-container-plugin"); - bundles.forEach(b -> addPlatformBundle(PlatformBundles.absoluteBundlePath(b))); + transport_connections_per_target = deployState.featureFlags().mbusJavaRpcNumTargets(); + transport_events_before_wakeup = deployState.featureFlags().mbusJavaEventsBeforeWakeup(); } @Override @@ -269,6 +264,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } if (getDocproc() != null) getDocproc().getConfig(builder); + builder.transport_events_before_wakeup(transport_events_before_wakeup); + builder.numconnectionspertarget(transport_connections_per_target); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 4b1c03a170c..63c323bfdaf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; @@ -148,19 +147,11 @@ public abstract class Container extends AbstractService implements return (parent instanceof ContainerCluster) ? ((ContainerCluster<?>) parent).getHttp() : null; } + @SuppressWarnings("unused") // used by amenders public JettyHttpServer getDefaultHttpServer() { return defaultHttpServer; } - public JettyHttpServer getHttpServer() { - Http http = getHttp(); - if (http == null) { - return defaultHttpServer; - } else { - return http.getHttpServer().orElse(null); - } - } - /** Returns the index of this node. The index of a given node is stable through changes with best effort. */ public final int index() { return index; } @@ -381,7 +372,7 @@ public abstract class Container extends AbstractService implements @Override public void getConfig(ContainerMbusConfig.Builder builder) { - builder.enabled(messageBusEnabled()).port(getMessagingPort()); + builder.port(getMessagingPort()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 5adea6d50e8..2385b5b3812 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -70,6 +70,8 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Parent class for all container cluster types. @@ -126,6 +128,10 @@ public abstract class ContainerCluster<CONTAINER extends Container> public static final BindingPattern VIP_HANDLER_BINDING = SystemBindingPattern.fromHttpPath("/status.html"); + public static final Set<Path> SEARCH_AND_DOCPROC_BUNDLES = Stream.of( + PlatformBundles.searchAndDocprocBundle, "container-search-gui", "docprocs", "linguistics-components") + .map(PlatformBundles::absoluteBundlePath).collect(Collectors.toSet()); + private final String name; protected List<CONTAINER> containers = new ArrayList<>(); @@ -388,6 +394,18 @@ public abstract class ContainerCluster<CONTAINER extends Container> return Collections.unmodifiableCollection(allComponents); } + /* + Add all search/docproc/feed related platform bundles. + This is only required for configured containers as the platform bundle set is not allowed to change between config generations. + For standalone container platform bundles can be added on features enabled as an update of application package requires restart. + */ + public void addAllPlatformBundles() { + ContainerDocumentApi.addVespaClientContainerBundle(this); + addSearchAndDocprocBundles(); + } + + public void addSearchAndDocprocBundles() { SEARCH_AND_DOCPROC_BUNDLES.forEach(this::addPlatformBundle); } + private void recursivelyFindAllComponents(Collection<Component<?, ?>> allComponents, AbstractConfigProducer<?> current) { for (AbstractConfigProducer<?> child: current.getChildren().values()) { if (child instanceof Component) @@ -412,8 +430,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> @Override public void getConfig(DocumentmanagerConfig.Builder builder) { - if (containerDocproc != null && containerDocproc.isCompressDocuments()) - builder.enablecompression(true); if (containerDocumentApi != null) builder.ignoreundefinedfields(containerDocumentApi.ignoreUndefinedFields()); } @@ -539,9 +555,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> @Override public void getConfig(IlscriptsConfig.Builder builder) { - List<SearchCluster> searchClusters = new ArrayList<>(); - searchClusters.addAll(Content.getSearchClusters(getRoot().configModelRepo())); - for (SearchCluster searchCluster : searchClusters) { + for (SearchCluster searchCluster : Content.getSearchClusters(getRoot().configModelRepo())) { searchCluster.getConfig(builder); } } @@ -613,8 +627,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> public void setEnvironmentVars(String environmentVars) { this.environmentVars = environmentVars; } - public String getEnvironmentVars() { return environmentVars; } - public Optional<String> getJvmGCOptions() { return Optional.ofNullable(jvmGCOptions); } public final void setRpcServerEnabled(boolean rpcServerEnabled) { this.rpcServerEnabled = rpcServerEnabled; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java index 8be02f77ab3..fee10b965aa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java @@ -27,29 +27,29 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> public final Options options; // Whether or not to prefer sending to a local node. - private boolean preferLocalNode = false; + private final boolean preferLocalNode = false; // The number of nodes to use per client. - private int numNodesPerClient = 0; + private final int numNodesPerClient = 0; - private Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>(); + private final Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>(); - public ContainerDocproc(ContainerCluster cluster, DocprocChains chains) { - this(cluster, chains, new Options(false, null, null, null, null, null, null)); + public ContainerDocproc(ContainerCluster<?> cluster, DocprocChains chains) { + this(cluster, chains, new Options( null, null, null, null, null, null)); } - public ContainerDocproc(ContainerCluster cluster, DocprocChains chains, Options options) { + public ContainerDocproc(ContainerCluster<?> cluster, DocprocChains chains, Options options) { this(cluster, chains, options, true); } private void addSource( - final ContainerCluster cluster, final String name, final SessionConfig.Type.Enum type) { + final ContainerCluster<?> cluster, final String name, final SessionConfig.Type.Enum type) { final MbusClient mbusClient = new MbusClient(name, type); mbusClient.addClientBindings(SystemBindingPattern.fromPattern("mbus://*/" + mbusClient.getSessionName())); cluster.addComponent(mbusClient); } - public ContainerDocproc(ContainerCluster cluster, DocprocChains chains, Options options, boolean addSourceClientProvider) { + public ContainerDocproc(ContainerCluster<?> cluster, DocprocChains chains, Options options, boolean addSourceClientProvider) { super(chains); assert (options != null) : "Null Options for " + this + " under cluster " + cluster.getName(); this.options = options; @@ -58,10 +58,7 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> addSource(cluster, "source", SessionConfig.Type.SOURCE); addSource(cluster, MbusRequestContext.internalNoThrottledSource, SessionConfig.Type.INTERNAL); } - } - - public boolean isCompressDocuments() { - return options.compressDocuments; + cluster.addSearchAndDocprocBundles(); } public boolean isPreferLocalNode() { @@ -75,8 +72,6 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> @Override public void getConfig(ContainerMbusConfig.Builder builder) { builder.maxpendingcount(getMaxMessagesInQueue()); - if (getMaxQueueMbSize() != null) - builder.maxpendingsize(getMaxQueueMbSize()); //yes, this shall be set in megabytes. } private int getMaxMessagesInQueue() { @@ -137,8 +132,6 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> } public static class Options { - // Whether or not to compress documents after processing them. - public final boolean compressDocuments; public final Integer maxMessagesInQueue; public final Integer maxQueueMbSize; @@ -148,8 +141,7 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> public final Double documentExpansionFactor; public final Integer containerCoreMemory; - public Options(boolean compressDocuments, Integer maxMessagesInQueue, Integer maxQueueMbSize, Integer maxQueueTimeMs, Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) { - this.compressDocuments = compressDocuments; + public Options(Integer maxMessagesInQueue, Integer maxQueueMbSize, Integer maxQueueTimeMs, Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) { this.maxMessagesInQueue = maxMessagesInQueue; this.maxQueueMbSize = maxQueueMbSize; this.maxQueueTimeMs = maxQueueTimeMs; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 58151063956..3ac12381a1f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -59,6 +59,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> owningCluster.addComponent(Component.fromClassAndBundle(QUERY_PROFILE_REGISTRY_CLASS, searchAndDocprocBundle)); owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class.getName(), searchAndDocprocBundle)); owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class.getName(), searchAndDocprocBundle)); + cluster.addSearchAndDocprocBundles(); } public void connectSearchClusters(Map<String, SearchCluster> searchClusters) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 7cc1109f25f..1c47f1d7c9c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -205,6 +205,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addServerProviders(deployState, spec, cluster); + if (!standaloneBuilder) cluster.addAllPlatformBundles(); + // Must be added after nodes: addDeploymentSpecConfig(cluster, context, deployState.getDeployLogger()); addZooKeeper(cluster, spec); @@ -595,6 +597,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Element processingElement = XML.getChild(spec, "processing"); if (processingElement == null) return; + cluster.addSearchAndDocprocBundles(); addIncludes(processingElement); cluster.setProcessingChains(new DomProcessingBuilder(null).build(deployState, cluster, processingElement), serverBindings(processingElement, ProcessingChains.defaultBindings).toArray(BindingPattern[]::new)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java index 1239cbf9bdc..e0dffca2bc5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java @@ -10,7 +10,6 @@ import org.w3c.dom.Element; public class DocprocOptionsBuilder { public static ContainerDocproc.Options build(Element spec) { return new ContainerDocproc.Options( - getCompression(spec), getMaxMessagesInQueue(spec), getSizeInMegabytes(spec.getAttribute("maxqueuebytesize")), getTime(spec.getAttribute("maxqueuewait")), @@ -25,10 +24,6 @@ public class DocprocOptionsBuilder { Integer.parseInt(integer); } - private static boolean getCompression(Element spec) { - return (spec.hasAttribute("compressdocuments") && spec.getAttribute("compressdocuments").equals("true")); - } - private static Double getFactor(String factor) { return factor == null || factor.trim().isEmpty() ? null : diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index d4595b3adfd..22923a724a6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -298,6 +298,7 @@ public class Content extends ConfigModel { content.ownedIndexingCluster = Optional.of(indexingCluster); indexingCluster.addDefaultHandlersWithVip(); + indexingCluster.addAllPlatformBundles(); addDocproc(indexingCluster); List<ApplicationContainer> nodes = new ArrayList<>(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java index 0b0a512a1c8..c52bb6fa2de 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java @@ -21,19 +21,21 @@ public abstract class ContentNode extends AbstractService private final int distributionKey; private final String rootDirectory; - private final boolean dispatch_on_encode; - private final boolean dispatch_on_decode; - private final int mbus_threads; private final int mbus_network_threads; + private final int mbus_rpc_targets; + private final int mbus_events_before_wakeup; + private final int rpc_num_targets; + private final int rpc_events_before_wakeup; public ContentNode(ModelContext.FeatureFlags featureFlags, AbstractConfigProducer<?> parent, String clusterName, String rootDirectory, int distributionKey) { super(parent, "" + distributionKey); this.distributionKey = distributionKey; this.rootDirectory = rootDirectory; - dispatch_on_decode = featureFlags.mbusDispatchOnDecode(); - dispatch_on_encode = featureFlags.mbusDispatchOnEncode(); - mbus_threads = featureFlags.mbusThreads(); mbus_network_threads = featureFlags.mbusNetworkThreads(); + mbus_rpc_targets = featureFlags.mbusCppRpcNumTargets(); + mbus_events_before_wakeup = featureFlags.mbusCppEventsBeforeWakeup(); + rpc_num_targets = featureFlags.rpcNumTargets(); + rpc_events_before_wakeup = featureFlags.rpcEventsBeforeWakeup(); initialize(); setProp("clustertype", "content"); @@ -77,10 +79,11 @@ public abstract class ContentNode extends AbstractService public void getConfig(StorCommunicationmanagerConfig.Builder builder) { builder.mbusport(getRelativePort(0)); builder.rpcport(getRelativePort(1)); - builder.mbus.dispatch_on_decode(dispatch_on_decode); - builder.mbus.dispatch_on_encode(dispatch_on_encode); - builder.mbus.num_threads(mbus_threads); builder.mbus.num_network_threads(mbus_network_threads); + builder.mbus.num_rpc_targets(mbus_rpc_targets); + builder.mbus.events_before_wakeup(mbus_events_before_wakeup); + builder.rpc.num_targets_per_node(rpc_num_targets); + builder.rpc.events_before_wakeup(rpc_events_before_wakeup); } @Override diff --git a/config-model/src/main/resources/schema/docproc.rnc b/config-model/src/main/resources/schema/docproc.rnc index 1e7e28b2002..42902f7180f 100644 --- a/config-model/src/main/resources/schema/docproc.rnc +++ b/config-model/src/main/resources/schema/docproc.rnc @@ -22,6 +22,7 @@ DocProcV3 = attribute version { "3.0" }, GenericConfig* ) +# TODO Here we need a thorough cleaning DocprocClusterAttributes = attribute compressdocuments { xsd:boolean }? & attribute numnodesperclient { xsd:positiveInteger }? & attribute preferlocalnode { xsd:boolean }? & @@ -32,6 +33,7 @@ DocprocClusterAttributes = attribute compressdocuments { xsd:boolean }? & attribute documentexpansionfactor { xsd:double { minExclusive = "0.0" } }? & attribute containercorememory { xsd:nonNegativeInteger }? +# TODO Here we need a thorough cleaning ClusterV3 = element cluster { attribute name { xsd:NCName } & DocprocClusterAttributes? & diff --git a/config-model/src/test/configmodel/types/documentmanager.cfg b/config-model/src/test/configmodel/types/documentmanager.cfg index c471935d9da..0269efcc4ca 100644 --- a/config-model/src/test/configmodel/types/documentmanager.cfg +++ b/config-model/src/test/configmodel/types/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/documenttypes.cfg b/config-model/src/test/configmodel/types/documenttypes.cfg index d35928a7a4b..b1c3075df2c 100644 --- a/config-model/src/test/configmodel/types/documenttypes.cfg +++ b/config-model/src/test/configmodel/types/documenttypes.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg index f8c7dacd3af..be4a18f2ab2 100644 --- a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg +++ b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg index bf2ae28c417..778fe2a44c4 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg index 7f93eea8e90..4d0a8396d91 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg index ecb499b5ed3..17d6264e138 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg index 6073faca616..9e76cb14b20 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg index 92bde94f54a..e8398b159dd 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false documenttype[].id 2987301 documenttype[].name "ad" diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg index 5ea07de4124..0d817fd4e07 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg index f31e5f6e7c0..f681a8de9ba 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/derived/advanced/documentmanager.cfg b/config-model/src/test/derived/advanced/documentmanager.cfg index 1ad50f57e93..375d42bc63e 100644 --- a/config-model/src/test/derived/advanced/documentmanager.cfg +++ b/config-model/src/test/derived/advanced/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg index 1ca66a7aea2..c5acbbf87e3 100644 --- a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg index 8dc07ae8eab..edf5fd13d1b 100644 --- a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg index 67c47032995..d931cae2048 100644 --- a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg index dbe6054ce6c..76882afd3c5 100644 --- a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg +++ b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationsreference/documentmanager.cfg b/config-model/src/test/derived/annotationsreference/documentmanager.cfg index d27ff5c9d07..ef0166ba4a7 100644 --- a/config-model/src/test/derived/annotationsreference/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsreference/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationssimple/documentmanager.cfg b/config-model/src/test/derived/annotationssimple/documentmanager.cfg index a52837c5c06..a076d5a7479 100644 --- a/config-model/src/test/derived/annotationssimple/documentmanager.cfg +++ b/config-model/src/test/derived/annotationssimple/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg index 4ab3f376d9d..d0b527ddaf3 100644 --- a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg index c2078dfa671..80792768c34 100644 --- a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/arrays/documentmanager.cfg b/config-model/src/test/derived/arrays/documentmanager.cfg index 836f3903079..ef3841f180b 100644 --- a/config-model/src/test/derived/arrays/documentmanager.cfg +++ b/config-model/src/test/derived/arrays/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg index a5c063108e4..5733c7d1bbd 100644 --- a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg +++ b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/complex/documentmanager.cfg b/config-model/src/test/derived/complex/documentmanager.cfg index dd0b7095d93..f44ab48c255 100644 --- a/config-model/src/test/derived/complex/documentmanager.cfg +++ b/config-model/src/test/derived/complex/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/declstruct/documentmanager.cfg b/config-model/src/test/derived/declstruct/documentmanager.cfg index 992d210dbe2..7b95c78d584 100644 --- a/config-model/src/test/derived/declstruct/documentmanager.cfg +++ b/config-model/src/test/derived/declstruct/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/duplicate_struct/documentmanager.cfg b/config-model/src/test/derived/duplicate_struct/documentmanager.cfg index 4742a75205a..af20f058eed 100644 --- a/config-model/src/test/derived/duplicate_struct/documentmanager.cfg +++ b/config-model/src/test/derived/duplicate_struct/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/duplicate_struct/documenttypes.cfg b/config-model/src/test/derived/duplicate_struct/documenttypes.cfg index 45c1bd8700f..b751b1ebbbf 100644 --- a/config-model/src/test/derived/duplicate_struct/documenttypes.cfg +++ b/config-model/src/test/derived/duplicate_struct/documenttypes.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/emptydefault/documentmanager.cfg b/config-model/src/test/derived/emptydefault/documentmanager.cfg index bf2a39df89f..63732728b05 100644 --- a/config-model/src/test/derived/emptydefault/documentmanager.cfg +++ b/config-model/src/test/derived/emptydefault/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/id/documentmanager.cfg b/config-model/src/test/derived/id/documentmanager.cfg index 33cdbe5b996..583ca8497b8 100644 --- a/config-model/src/test/derived/id/documentmanager.cfg +++ b/config-model/src/test/derived/id/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg index c6cd1a2949d..aa960e70326 100644 --- a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg +++ b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/indexswitches/documentmanager.cfg b/config-model/src/test/derived/indexswitches/documentmanager.cfg index fa91cb0e554..a902a5954a5 100644 --- a/config-model/src/test/derived/indexswitches/documentmanager.cfg +++ b/config-model/src/test/derived/indexswitches/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/inheritance/documentmanager.cfg b/config-model/src/test/derived/inheritance/documentmanager.cfg index 52cac7bfa79..fd4994dbefb 100644 --- a/config-model/src/test/derived/inheritance/documentmanager.cfg +++ b/config-model/src/test/derived/inheritance/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/inheritdiamond/documentmanager.cfg b/config-model/src/test/derived/inheritdiamond/documentmanager.cfg index 7d5dacbe00f..9f1207b77a3 100644 --- a/config-model/src/test/derived/inheritdiamond/documentmanager.cfg +++ b/config-model/src/test/derived/inheritdiamond/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg index 537c452c38c..68503ebd5aa 100644 --- a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg index d246e886a3d..c8fdeaedf11 100644 --- a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg index efd8170d95e..fd9623f716a 100644 --- a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg +++ b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/mail/onlydoc/documentmanager.cfg b/config-model/src/test/derived/mail/onlydoc/documentmanager.cfg index 27ac015e630..6bf972c1c80 100644 --- a/config-model/src/test/derived/mail/onlydoc/documentmanager.cfg +++ b/config-model/src/test/derived/mail/onlydoc/documentmanager.cfg @@ -1,4 +1,3 @@ -enablecompression false usev8geopositions false datatype[].id 1381038251 datatype[].structtype[].name "position" diff --git a/config-model/src/test/derived/multi_struct/documentmanager.cfg b/config-model/src/test/derived/multi_struct/documentmanager.cfg index e37a3dc51c6..753196b4d02 100644 --- a/config-model/src/test/derived/multi_struct/documentmanager.cfg +++ b/config-model/src/test/derived/multi_struct/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/multi_struct/documenttypes.cfg b/config-model/src/test/derived/multi_struct/documenttypes.cfg index 93452602f86..4926675582b 100644 --- a/config-model/src/test/derived/multi_struct/documenttypes.cfg +++ b/config-model/src/test/derived/multi_struct/documenttypes.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/namecollision/documentmanager.cfg b/config-model/src/test/derived/namecollision/documentmanager.cfg index d8cf44a9a3d..79807197b48 100644 --- a/config-model/src/test/derived/namecollision/documentmanager.cfg +++ b/config-model/src/test/derived/namecollision/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg index d516eaf7886..3158348d5be 100644 --- a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg +++ b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/ranktypes/documentmanager.cfg b/config-model/src/test/derived/ranktypes/documentmanager.cfg index 46457fb479d..ade7795c3ca 100644 --- a/config-model/src/test/derived/ranktypes/documentmanager.cfg +++ b/config-model/src/test/derived/ranktypes/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/reference_from_several/documentmanager.cfg b/config-model/src/test/derived/reference_from_several/documentmanager.cfg index 28f40aeee5b..bdcb6a04236 100644 --- a/config-model/src/test/derived/reference_from_several/documentmanager.cfg +++ b/config-model/src/test/derived/reference_from_several/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/schemainheritance/documentmanager.cfg b/config-model/src/test/derived/schemainheritance/documentmanager.cfg index 1fe61cf2bd2..a706c57909e 100644 --- a/config-model/src/test/derived/schemainheritance/documentmanager.cfg +++ b/config-model/src/test/derived/schemainheritance/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/streamingstruct/documentmanager.cfg b/config-model/src/test/derived/streamingstruct/documentmanager.cfg index b94f23a9c7b..c29ff0b3489 100644 --- a/config-model/src/test/derived/streamingstruct/documentmanager.cfg +++ b/config-model/src/test/derived/streamingstruct/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/structandfieldset/documentmanager.cfg b/config-model/src/test/derived/structandfieldset/documentmanager.cfg index e1169e3ca5d..e9bbc06d77a 100644 --- a/config-model/src/test/derived/structandfieldset/documentmanager.cfg +++ b/config-model/src/test/derived/structandfieldset/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/structanyorder/documentmanager.cfg b/config-model/src/test/derived/structanyorder/documentmanager.cfg index eac63515944..a1e8bb41a97 100644 --- a/config-model/src/test/derived/structanyorder/documentmanager.cfg +++ b/config-model/src/test/derived/structanyorder/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/structinheritance/documentmanager.cfg b/config-model/src/test/derived/structinheritance/documentmanager.cfg index 37240887e3b..5897b00c07b 100644 --- a/config-model/src/test/derived/structinheritance/documentmanager.cfg +++ b/config-model/src/test/derived/structinheritance/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/structinheritance/documenttypes.cfg b/config-model/src/test/derived/structinheritance/documenttypes.cfg index 16521b920a7..3f9fdf85734 100644 --- a/config-model/src/test/derived/structinheritance/documenttypes.cfg +++ b/config-model/src/test/derived/structinheritance/documenttypes.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/tensor/documentmanager.cfg b/config-model/src/test/derived/tensor/documentmanager.cfg index f52fe073208..bae2db34040 100644 --- a/config-model/src/test/derived/tensor/documentmanager.cfg +++ b/config-model/src/test/derived/tensor/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/tensor/documenttypes.cfg b/config-model/src/test/derived/tensor/documenttypes.cfg index 3081a5bd0c4..d10ecd37c8f 100644 --- a/config-model/src/test/derived/tensor/documenttypes.cfg +++ b/config-model/src/test/derived/tensor/documenttypes.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/types/documentmanager.cfg b/config-model/src/test/derived/types/documentmanager.cfg index 118f7d279e6..ace183e1ab0 100644 --- a/config-model/src/test/derived/types/documentmanager.cfg +++ b/config-model/src/test/derived/types/documentmanager.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg b/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg index 2efc2f40d21..ebc72c2d73b 100644 --- a/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg +++ b/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/examples/fieldoftypedocument.cfg b/config-model/src/test/examples/fieldoftypedocument.cfg index 10c66ce3e93..5aca758a5f4 100644 --- a/config-model/src/test/examples/fieldoftypedocument.cfg +++ b/config-model/src/test/examples/fieldoftypedocument.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/examples/structresult.cfg b/config-model/src/test/examples/structresult.cfg index 2e3904b7110..c2c84eb46b0 100644 --- a/config-model/src/test/examples/structresult.cfg +++ b/config-model/src/test/examples/structresult.cfg @@ -1,5 +1,4 @@ ignoreundefinedfields false -enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 198b5713876..b634356fcb6 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -162,15 +162,15 @@ public class ContainerClusterTest { addContainer(root, cluster, "c1", "host-c1"); assertEquals(1, cluster.getContainers().size()); ApplicationContainer container = cluster.getContainers().get(0); - verifyJvmArgs(isHosted, hasDocProc, "", container.getJvmOptions()); + verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, ""), container.getJvmOptions()); container.setJvmOptions("initial"); - verifyJvmArgs(isHosted, hasDocProc, "initial", container.getJvmOptions()); + verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, "initial"), container.getJvmOptions()); container.prependJvmOptions("ignored"); - verifyJvmArgs(isHosted, hasDocProc, "ignored initial", container.getJvmOptions()); + verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, "ignored initial"), container.getJvmOptions()); container.appendJvmOptions("override"); - verifyJvmArgs(isHosted, hasDocProc, "ignored initial override", container.getJvmOptions()); + verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, "ignored initial override"), container.getJvmOptions()); container.setJvmOptions(null); - verifyJvmArgs(isHosted, hasDocProc, "", container.getJvmOptions()); + verifyJvmArgs(isHosted, hasDocProc, expectedJvmArgs(isHosted, ""), container.getJvmOptions()); } @Test @@ -510,4 +510,13 @@ public class ContainerClusterTest { return new ClusterInfoConfig(builder); } + private static String expectedJvmArgs(boolean isHosted, String extra) { + if (!isHosted) return extra; + return "-Djdk.tls.server.enableStatusRequestExtension=true " + + "-Djdk.tls.stapling.responseTimeout=2000 " + + "-Djdk.tls.stapling.cacheSize=256 " + + "-Djdk.tls.stapling.cacheLifetime=3600" + + (extra.isEmpty() ? "" : " " + extra); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/docproc/StandaloneDocprocContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/docproc/StandaloneDocprocContainerTest.java deleted file mode 100644 index 5bb93255a8f..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/docproc/StandaloneDocprocContainerTest.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.docproc; - -import com.yahoo.component.ComponentId; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.builder.xml.test.DomBuilderTest; -import com.yahoo.vespa.model.container.ContainerCluster; -import com.yahoo.vespa.model.container.ContainerModel; -import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; -import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking; -import org.junit.Test; -import org.w3c.dom.Element; - -import java.util.Map; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author Einar M R Rosenvinge - */ -public class StandaloneDocprocContainerTest extends DomBuilderTest { - - public ContainerCluster setupCluster(boolean standalone) { - ContainerModelBuilder builder = new ContainerModelBuilder(standalone, Networking.disable); - ContainerModel model = builder.build(DeployState.createTestState(), null, null, root, servicesXml()); - - if (!standalone) - model.getCluster().getDocproc().getChains().addServersAndClientsForChains(); - - root.freezeModelTopology(); - return model.getCluster(); - } - - private Element servicesXml() { - return parse("" + - "<container version=\"1.0\">\n" + - " <document-processing>\n" + - " <chain id=\"foo\">\n" + - " <documentprocessor id=\"MyDocproc\"/>\n" + - " </chain>\n" + - " </document-processing>\n" + - " <nodes>\n" + - " <node hostalias=\"node01\"/>\n" + - " </nodes>\n" + - "</container>\n"); - } - - @Test - public void requireMbusProvidersWhenNonStandalone() { - ContainerCluster containerCluster = setupCluster(false); - Map<ComponentId, Component<?, ?>> components = containerCluster.getComponentsMap(); - - boolean foundAtLeastOneClient = false; - boolean foundAtLeastOneServer = false; - - for (ComponentId componentId : components.keySet()) { - if (componentId.stringValue().contains("MbusClient")) foundAtLeastOneClient = true; - if (componentId.stringValue().contains("MbusServer")) foundAtLeastOneServer = true; - } - assertTrue(foundAtLeastOneClient); - assertTrue(foundAtLeastOneServer); - - } - - @Test - public void requireNoMbusProvidersWhenStandalone() { - ContainerCluster containerCluster = setupCluster(true); - Map<ComponentId, Component<?, ?>> components = containerCluster.getComponentsMap(); - - boolean foundAtLeastOneClient = false; - boolean foundAtLeastOneServer = false; - - for (ComponentId componentId : components.keySet()) { - if (componentId.stringValue().contains("MbusClient")) foundAtLeastOneClient = true; - if (componentId.stringValue().contains("MbusServer")) foundAtLeastOneServer = true; - } - assertFalse(foundAtLeastOneClient); - assertFalse(foundAtLeastOneServer); - } -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java index 77681489dac..2044fd2ab39 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -1,14 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; -import com.yahoo.config.docproc.DocprocConfig; import com.yahoo.config.docproc.SchemamappingConfig; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.jdisc.ContainerMbusConfig; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.HostPorts; import com.yahoo.vespa.model.container.ApplicationContainer; @@ -38,12 +36,10 @@ import static org.junit.Assert.assertTrue; public class DocprocBuilderTest extends DomBuilderTest { private ApplicationContainerCluster cluster; - private DocumentmanagerConfig documentmanagerConfig; private ContainerMbusConfig containerMbusConfig; private ComponentsConfig componentsConfig; private ChainsConfig chainsConfig; private SchemamappingConfig schemamappingConfig; - private DocprocConfig docprocConfig; private QrStartConfig qrStartConfig; @Before @@ -58,10 +54,8 @@ public class DocprocBuilderTest extends DomBuilderTest { chainsConfig = root.getConfig(ChainsConfig.class, cluster.getConfigId() + "/component/com.yahoo.docproc.jdisc.DocumentProcessingHandler"); - documentmanagerConfig = root.getConfig(DocumentmanagerConfig.class, cluster.getConfigId()); schemamappingConfig = root.getConfig(SchemamappingConfig.class, cluster.getContainers().get(0).getConfigId()); qrStartConfig = root.getConfig(QrStartConfig.class, cluster.getConfigId()); - docprocConfig = root.getConfig(DocprocConfig.class, cluster.getConfigId()); } private Element servicesXml() { @@ -70,7 +64,7 @@ public class DocprocBuilderTest extends DomBuilderTest { " <nodes>", " <node hostalias='mockhost' baseport='1500' />", " </nodes>", - " <document-processing compressdocuments='true' preferlocalnode='true' numnodesperclient='2' maxqueuebytesize='100m' maxmessagesinqueue='300' maxqueuewait='200'>", + " <document-processing preferlocalnode='true' numnodesperclient='2' maxqueuebytesize='100m' maxmessagesinqueue='300' maxqueuewait='200'>", " <documentprocessor id='docproc1' class='com.yahoo.Docproc1' bundle='docproc1bundle'/>", " <chain id='chein'>", " <documentprocessor id='docproc2'/>", @@ -83,7 +77,6 @@ public class DocprocBuilderTest extends DomBuilderTest { @Test public void testDocprocCluster() { assertEquals("banan", cluster.getName()); - assertTrue(cluster.getDocproc().isCompressDocuments()); //assertTrue(cluster.getContainerDocproc().isPreferLocalNode()); //assertEquals(2, cluster.getContainerDocproc().getNumNodesPerClient()); List<ApplicationContainer> services = cluster.getContainers(); @@ -105,16 +98,9 @@ public class DocprocBuilderTest extends DomBuilderTest { } @Test - public void testDocumentManagerConfig() { - assertTrue(documentmanagerConfig.enablecompression()); - } - - @Test public void testContainerMbusConfig() { - assertTrue(containerMbusConfig.enabled()); assertTrue(containerMbusConfig.port() >= HostPorts.BASE_PORT); assertEquals(300, containerMbusConfig.maxpendingcount()); - assertEquals(100, containerMbusConfig.maxpendingsize()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index 2031e74bd5b..015356d6088 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -273,7 +273,6 @@ public class DistributorTest { cluster.getChildren().get("0").getConfig(builder); StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); - assertTrue(config.mbus().dispatch_on_encode()); assertEquals(14066, config.rpcport()); } @@ -290,9 +289,6 @@ public class DistributorTest { cluster.getChildren().get("0").getConfig(builder); StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); - assertTrue(config.mbus().dispatch_on_encode()); - assertTrue(config.mbus().dispatch_on_decode()); - assertEquals(4, config.mbus().num_threads()); assertEquals(1, config.mbus().num_network_threads()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 5748f260bf8..c494ba0394a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -101,9 +101,6 @@ public class StorageClusterTest { StorCommunicationmanagerConfig.Builder builder = new StorCommunicationmanagerConfig.Builder(); storage.getChildren().get("0").getConfig(builder); StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); - assertTrue(config.mbus().dispatch_on_encode()); - assertTrue(config.mbus().dispatch_on_decode()); - assertEquals(4, config.mbus().num_threads()); assertEquals(1, config.mbus().num_network_threads()); } @@ -154,23 +151,26 @@ public class StorageClusterTest { @Test public void verifyDefaultMbusConfig() { var confg = communicationmanagerConfigFromProperties(new TestProperties()); - assertTrue(confg.mbus().dispatch_on_decode()); - assertTrue(confg.mbus().dispatch_on_encode()); - assertEquals(4, confg.mbus().num_threads()); assertEquals(1, confg.mbus().num_network_threads()); + assertEquals(1, confg.mbus().num_rpc_targets()); + assertEquals(1, confg.mbus().events_before_wakeup()); + assertEquals(1, confg.rpc().num_targets_per_node()); + assertEquals(1, confg.rpc().events_before_wakeup()); } @Test public void verifyDefaultMbusConfigControl() { var confg = communicationmanagerConfigFromProperties(new TestProperties() - .setMbusDispatchOnDecode(false) - .setMbusDispatchOnEncode(false) - .setMbusThreads(3) - .setMbusNetworkThreads(7)); - assertFalse(confg.mbus().dispatch_on_decode()); - assertFalse(confg.mbus().dispatch_on_encode()); - assertEquals(3, confg.mbus().num_threads()); + .setMbusNetworkThreads(7) + .setRpcNumTargets(11) + .setRpcEventsBeforeWakeup(12) + .setMbusCppRpcNumTargets(8) + .setMbusCppEventsBeforeWakeup(9)); assertEquals(7, confg.mbus().num_network_threads()); + assertEquals(8, confg.mbus().num_rpc_targets()); + assertEquals(9, confg.mbus().events_before_wakeup()); + assertEquals(11, confg.rpc().num_targets_per_node()); + assertEquals(12, confg.rpc().events_before_wakeup()); } @Test diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java index 68570722117..edd16c3d23d 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java @@ -6,11 +6,15 @@ import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Supervisor; import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool; import com.yahoo.vespa.filedistribution.FileDownloader; - import java.time.Duration; +import java.util.Arrays; +import java.util.Set; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; /** * Keeps track of file distribution and url download rpc servers. @@ -45,9 +49,16 @@ public class FileDistributionAndUrlDownload { } private FileDownloader createDownloader(Supervisor supervisor, ConfigSourceSet source) { + Set<CompressionType> acceptedCompressionTypes = Set.of(CompressionType.gzip); + String env = System.getenv("VESPA_FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES"); + if (env != null && ! env.isEmpty()) { + String[] types = env.split(","); + acceptedCompressionTypes = Arrays.stream(types).map(CompressionType::valueOf).collect(Collectors.toSet()); + } return new FileDownloader(new FileDistributionConnectionPool(source, supervisor), supervisor, - Duration.ofMinutes(5)); + Duration.ofMinutes(5), + acceptedCompressionTypes); } } diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java b/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java index aabfd211fac..316edda1ee8 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; - import com.yahoo.config.ConfigInstance; /** @@ -13,7 +12,9 @@ import com.yahoo.config.ConfigInstance; * command-line tools. * * @author gjoranv + * @deprecated Use config builders where possible */ +@Deprecated public class ConfigGetter<T extends ConfigInstance> { private final Class<T> clazz; diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java index fb116729640..68b5d6a37d3 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java @@ -12,6 +12,7 @@ import static org.junit.Assert.assertTrue; * * @author gjoranv */ +@SuppressWarnings("deprecation") public class ConfigGetterTest { private final ConfigSourceSet sourceSet = new ConfigSourceSet("config-getter-test"); diff --git a/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java b/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java index 42c2c599899..8656c0e945f 100644 --- a/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java @@ -2,20 +2,16 @@ package com.yahoo.config.subscription; import com.yahoo.foo.FunctionTestConfig; - import org.junit.Before; import org.junit.Test; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -28,6 +24,7 @@ import static org.junit.Assert.fail; * * @author gjoranv */ +@SuppressWarnings("deprecation") public class FunctionTest { public static final String PATH = "src/test/resources/configs/function-test/"; diff --git a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java index 963f3de5e43..2e7beb0f0f4 100644 --- a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java @@ -12,6 +12,7 @@ import static org.junit.Assert.assertEquals; public class NamespaceTest { @Test + @SuppressWarnings("deprecation") public void verifyConfigClassWithExplicitNamespace() { NamespaceConfig config = new ConfigGetter<>(NamespaceConfig.class).getConfig("raw: a 0\n"); assertEquals(0, config.a()); diff --git a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java index 8864f114d87..c1d98ac7e3e 100644 --- a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java @@ -22,6 +22,7 @@ public class UnicodeTest { * received correctly from the server */ @Test + @SuppressWarnings("deprecation") public void testUnicodeConfigReading() { ConfigGetter<UnicodeConfig> getter = new ConfigGetter<>(UnicodeConfig.class); UnicodeConfig config = getter.getConfig("file:src/test/resources/configs/unicode/unicode.cfg"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 47d1193cd4c..068323f7784 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -197,7 +197,6 @@ public class ModelContextImpl implements ModelContext { private final boolean useV8GeoPositions; private final int maxCompactBuffers; private final List<String> ignoredHttpUserAgents; - private final boolean enableServerOcspStapling; private final String mergeThrottlingPolicy; private final double persistenceThrottlingWsDecrementFactor; private final double persistenceThrottlingWsBackoff; @@ -215,6 +214,12 @@ public class ModelContextImpl implements ModelContext { private final boolean mbus_dispatch_on_encode; private final int mbus_threads; private final int mbus_network_threads; + private int mbus_java_num_targets; + private int mbus_java_events_before_wakeup; + private int mbus_cpp_num_targets; + private int mbus_cpp_events_before_wakeup; + private int rpc_num_targets; + private int rpc_events_before_wakeup; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -252,7 +257,6 @@ public class ModelContextImpl implements ModelContext { this.useV8GeoPositions = flagValue(source, appId, version, Flags.USE_V8_GEO_POSITIONS); this.maxCompactBuffers = flagValue(source, appId, version, Flags.MAX_COMPACT_BUFFERS); this.ignoredHttpUserAgents = flagValue(source, appId, version, PermanentFlags.IGNORED_HTTP_USER_AGENTS); - this.enableServerOcspStapling = flagValue(source, appId, version, Flags.ENABLE_SERVER_OCSP_STAPLING); this.mergeThrottlingPolicy = flagValue(source, appId, version, Flags.MERGE_THROTTLING_POLICY); this.persistenceThrottlingWsDecrementFactor = flagValue(source, appId, version, Flags.PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR); this.persistenceThrottlingWsBackoff = flagValue(source, appId, version, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF); @@ -266,6 +270,12 @@ public class ModelContextImpl implements ModelContext { this.enableProxyProtocolMixedMode = flagValue(source, appId, version, Flags.ENABLE_PROXY_PROTOCOL_MIXED_MODE); this.sharedStringRepoNoReclaim = flagValue(source, appId, version, Flags.SHARED_STRING_REPO_NO_RECLAIM); this.logFileCompressionAlgorithm = flagValue(source, appId, version, Flags.LOG_FILE_COMPRESSION_ALGORITHM); + this.mbus_java_num_targets = flagValue(source, appId, version, Flags.MBUS_JAVA_NUM_TARGETS); + this.mbus_java_events_before_wakeup = flagValue(source, appId, version, Flags.MBUS_JAVA_EVENTS_BEFORE_WAKEUP); + this.mbus_cpp_num_targets = flagValue(source, appId, version, Flags.MBUS_CPP_NUM_TARGETS); + this.mbus_cpp_events_before_wakeup = flagValue(source, appId, version, Flags.MBUS_CPP_EVENTS_BEFORE_WAKEUP); + this.rpc_num_targets = flagValue(source, appId, version, Flags.RPC_NUM_TARGETS); + this.rpc_events_before_wakeup = flagValue(source, appId, version, Flags.RPC_EVENTS_BEFORE_WAKEUP); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -305,7 +315,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public List<String> ignoredHttpUserAgents() { return ignoredHttpUserAgents; } - @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; } @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } @@ -318,6 +327,12 @@ public class ModelContextImpl implements ModelContext { @Override public Architecture adminClusterArchitecture() { return adminClusterArchitecture; } @Override public boolean enableProxyProtocolMixedMode() { return enableProxyProtocolMixedMode; } @Override public boolean sharedStringRepoNoReclaim() { return sharedStringRepoNoReclaim; } + @Override public int mbusJavaRpcNumTargets() { return mbus_java_num_targets; } + @Override public int mbusJavaEventsBeforeWakeup() { return mbus_java_events_before_wakeup; } + @Override public int mbusCppRpcNumTargets() { return mbus_cpp_num_targets; } + @Override public int mbusCppEventsBeforeWakeup() { return mbus_cpp_events_before_wakeup; } + @Override public int rpcNumTargets() { return rpc_num_targets; } + @Override public int rpcEventsBeforeWakeup() { return rpc_events_before_wakeup; } @Override public String logFileCompressionAlgorithm(String defVal) { var fflag = this.logFileCompressionAlgorithm; if (fflag != null && ! fflag.equals("")) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index d3b7e53157d..7f120a88a05 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -141,7 +141,7 @@ public class FileDirectory { File destination = new File(tempDestinationDir.toFile(), source.getName()); if (!destinationDir.exists()) { destinationDir.mkdir(); - log.log(Level.FINE, () -> "file reference ' " + reference.value() + "', source: " + source.getAbsolutePath() ); + log.log(Level.FINE, () -> "file reference '" + reference.value() + "', source: " + source.getAbsolutePath() ); if (source.isDirectory()) { log.log(Level.FINE, () -> "Copying source " + source.getAbsolutePath() + " to " + destination.getAbsolutePath()); IOUtils.copyDirectory(source, destination, -1); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index f0c34e83713..770352e6bfc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -21,6 +21,8 @@ import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDownload; import com.yahoo.vespa.filedistribution.LazyFileReferenceData; import com.yahoo.vespa.filedistribution.LazyTemporaryStorageFileReferenceData; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; @@ -28,6 +30,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -35,12 +38,13 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getOtherConfigServersInCluster; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; -import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4; import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type; public class FileServer { @@ -52,6 +56,7 @@ public class FileServer { private final FileDirectory root; private final ExecutorService executor; private final FileDownloader downloader; + private final List<CompressionType> compressionTypes; // compression types to use, in preferred order // TODO: Move to filedistribution module, so that it can be used by both clients and servers private enum FileApiErrorCodes { @@ -86,21 +91,24 @@ public class FileServer { @SuppressWarnings("WeakerAccess") // Created by dependency injection @Inject - public FileServer(ConfigserverConfig configserverConfig) { + public FileServer(ConfigserverConfig configserverConfig, FlagSource flagSource) { this(new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())), - createFileDownloader(getOtherConfigServersInCluster(configserverConfig))); + createFileDownloader(getOtherConfigServersInCluster(configserverConfig), + compressionTypes(Flags.FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES.bindTo(flagSource).value())), + compressionTypesAsList(Flags.FILE_DISTRIBUTION_COMPRESSION_TYPES_TO_SERVE.bindTo(flagSource).value())); } // For testing only public FileServer(File rootDir) { - this(rootDir, createFileDownloader(List.of())); + this(rootDir, createFileDownloader(List.of(), Set.of(gzip)), List.of(gzip)); } - public FileServer(File rootDir, FileDownloader fileDownloader) { + FileServer(File rootDir, FileDownloader fileDownloader, List<CompressionType> compressionTypes) { this.downloader = fileDownloader; this.root = new FileDirectory(rootDir); this.executor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), new DaemonThreadFactory("file-server-")); + this.compressionTypes = compressionTypes; } boolean hasFile(String fileReference) { @@ -145,10 +153,11 @@ public class FileServer { if (file.isDirectory()) { Path tempFile = Files.createTempFile("filereferencedata", reference.value()); CompressionType compressionType = chooseCompressionType(acceptedCompressionTypes); + log.log(Level.FINE, () -> "accepted compression types=" + acceptedCompressionTypes + ", compression type to use=" + compressionType); File compressedFile = new FileReferenceCompressor(compressed, compressionType).compress(file.getParentFile(), tempFile.toFile()); - return new LazyTemporaryStorageFileReferenceData(reference, file.getName(), compressed, compressedFile); + return new LazyTemporaryStorageFileReferenceData(reference, file.getName(), compressed, compressedFile, compressionType); } else { - return new LazyFileReferenceData(reference, file.getName(), FileReferenceData.Type.file, file); + return new LazyFileReferenceData(reference, file.getName(), Type.file, file, gzip); } } @@ -195,9 +204,14 @@ public class FileServer { return (fileExists ? FileApiErrorCodes.OK : FileApiErrorCodes.NOT_FOUND); } - // TODO: Use lz4 for testing only, add zstd when we have support for (de)compressing zstd input and output streams + /* Choose the first compression type (list is in preferred order) that matches an accepted compression type, or fail */ private CompressionType chooseCompressionType(Set<CompressionType> acceptedCompressionTypes) { - return acceptedCompressionTypes.contains(lz4) ? lz4 : gzip; + for (CompressionType compressionType : compressionTypes) { + if (acceptedCompressionTypes.contains(compressionType)) + return compressionType; + } + throw new RuntimeException("Could not find a compression type that can be used. Accepted compression types: " + + acceptedCompressionTypes + ", compression types server can use: " + compressionTypes); } boolean hasFileDownloadIfNeeded(FileReferenceDownload fileReferenceDownload) { @@ -228,14 +242,27 @@ public class FileServer { executor.shutdown(); } - private static FileDownloader createFileDownloader(List<String> configServers) { + private static FileDownloader createFileDownloader(List<String> configServers, Set<CompressionType> acceptedCompressionTypes) { Supervisor supervisor = new Supervisor(new Transport("filedistribution-pool")).setDropEmptyBuffers(true); return new FileDownloader(configServers.isEmpty() ? FileDownloader.emptyConnectionPool() : createConnectionPool(configServers, supervisor), supervisor, - timeout); + timeout, + acceptedCompressionTypes); + } + + private static LinkedHashSet<CompressionType> compressionTypes(List<String> compressionTypes) { + return compressionTypes.stream() + .map(CompressionType::valueOf) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + private static List<CompressionType> compressionTypesAsList(List<String> compressionTypes) { + return compressionTypes.stream() + .map(CompressionType::valueOf) + .collect(Collectors.toList()); } private static ConnectionPool createConnectionPool(List<String> configServers, Supervisor supervisor) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java index f7042b49c3f..0d4baa7dc56 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java @@ -38,4 +38,9 @@ class ProxyResponse extends HttpResponse { } } + @Override + public long maxPendingBytes() { + return 1 << 25; // 32MB + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index ae4b205c06e..12972e5c465 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -18,20 +18,23 @@ import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool; import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceDownload; import com.yahoo.vespa.flags.FlagSource; - +import com.yahoo.vespa.flags.Flags; import java.io.File; import java.time.Duration; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getOtherConfigServersInCluster; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; /** * Verifies that all active sessions has an application package on local disk. * If not, the package is downloaded with file distribution. This can happen e.g. - * if a configserver is down when the application is deployed. + * if a config server is down when the application is deployed. * * @author gjoranv */ @@ -53,7 +56,10 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { this.applicationRepository = applicationRepository; this.configserverConfig = applicationRepository.configserverConfig(); this.downloadDirectory = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())); - this.fileDownloader = createFileDownloader(configserverConfig, downloadDirectory, supervisor); + this.fileDownloader = createFileDownloader(configserverConfig, + downloadDirectory, + supervisor, + Flags.FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES.bindTo(flagSource).value()); } @Override @@ -94,14 +100,18 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { private static FileDownloader createFileDownloader(ConfigserverConfig configserverConfig, File downloadDirectory, - Supervisor supervisor) { + Supervisor supervisor, + List<String> flagValues) { List<String> otherConfigServersInCluster = getOtherConfigServersInCluster(configserverConfig); ConfigSourceSet configSourceSet = new ConfigSourceSet(otherConfigServersInCluster); ConnectionPool connectionPool = (otherConfigServersInCluster.isEmpty()) ? FileDownloader.emptyConnectionPool() : new FileDistributionConnectionPool(configSourceSet, supervisor); - return new FileDownloader(connectionPool, supervisor, downloadDirectory, Duration.ofSeconds(300)); + Set<CompressionType> acceptedCompressionTypes = flagValues.stream() + .map(CompressionType::valueOf) + .collect(Collectors.toSet()); + return new FileDownloader(connectionPool, supervisor, downloadDirectory, Duration.ofSeconds(300), acceptedCompressionTypes); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 687cb1d3cca..7993e58d06e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -491,6 +491,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { sendParts(session, fileData); sendEof(session, fileData, status); } + private void sendParts(int session, FileReferenceData fileData) { ByteBuffer bb = ByteBuffer.allocate(0x100000); for (int partId = 0, read = fileData.nextContent(bb); read >= 0; partId++, read = fileData.nextContent(bb)) { @@ -504,12 +505,9 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { bb.clear(); } } + private int sendMeta(FileReferenceData fileData) { - Request request = new Request(FileReceiver.RECEIVE_META_METHOD); - request.parameters().add(new StringValue(fileData.fileReference().value())); - request.parameters().add(new StringValue(fileData.filename())); - request.parameters().add(new StringValue(fileData.type().name())); - request.parameters().add(new Int64Value(fileData.size())); + Request request = createMetaRequest(fileData); invokeRpcIfValidConnection(request); if (request.isError()) { log.warning("Failed delivering meta for reference '" + fileData.fileReference().value() + "' with file '" + fileData.filename() + "' to " + @@ -522,6 +520,20 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { return request.returnValues().get(1).asInt32(); } } + + // non-private for testing + static Request createMetaRequest(FileReferenceData fileData) { + Request request = new Request(FileReceiver.RECEIVE_META_METHOD); + request.parameters().add(new StringValue(fileData.fileReference().value())); + request.parameters().add(new StringValue(fileData.filename())); + request.parameters().add(new StringValue(fileData.type().name())); + request.parameters().add(new Int64Value(fileData.size())); + // Only add paramter if not gzip, this is default and old clients will not handle the extra parameter + if (fileData.compressionType() != CompressionType.gzip) + request.parameters().add(new StringValue(fileData.compressionType().name())); + return request; + } + private void sendPart(int session, FileReference ref, int partId, byte [] buf) { Request request = new Request(FileReceiver.RECEIVE_PART_METHOD); request.parameters().add(new StringValue(ref.value())); @@ -538,6 +550,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } } } + private void sendEof(int session, FileReferenceData fileData, FileServer.ReplayStatus status) { Request request = new Request(FileReceiver.RECEIVE_EOF_METHOD); request.parameters().add(new StringValue(fileData.fileReference().value())); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index a144940e443..d803488cb0a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -44,13 +44,11 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.StringFlag; import com.yahoo.yolean.Exceptions; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.zookeeper.KeeperException; - import java.io.File; import java.io.FilenameFilter; import java.io.IOException; @@ -83,6 +81,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import static com.yahoo.vespa.curator.Curator.CompletionWaiter; +import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; import static java.nio.file.Files.readAttributes; /** @@ -128,7 +127,6 @@ public class SessionRepository { private final ModelFactoryRegistry modelFactoryRegistry; private final ConfigDefinitionRepo configDefinitionRepo; private final int maxNodeSize; - private final StringFlag failDeploymentForFilesWithUnknownExtension; public SessionRepository(TenantName tenantName, TenantApplications applicationRepo, @@ -172,7 +170,6 @@ public class SessionRepository { this.modelFactoryRegistry = modelFactoryRegistry; this.configDefinitionRepo = configDefinitionRepo; this.maxNodeSize = maxNodeSize; - this.failDeploymentForFilesWithUnknownExtension = Flags.APPLICATION_FILES_WITH_UNKNOWN_EXTENSION.bindTo(flagSource); loadSessions(); // Needs to be done before creating cache below this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, zkCacheExecutor); @@ -684,7 +681,10 @@ public class SessionRepository { try { app.validateFileExtensions(); } catch (IllegalArgumentException e) { - switch (failDeploymentForFilesWithUnknownExtension.value()) { + String flag = Flags.APPLICATION_FILES_WITH_UNKNOWN_EXTENSION.bindTo(flagSource) + .with(APPLICATION_ID, applicationId.serializedForm()) + .value(); + switch (flag) { case "FAIL": throw e; case "LOG": diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java index 9498db1d1e0..39219471bb1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java @@ -8,8 +8,10 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.net.HostName; import com.yahoo.vespa.filedistribution.FileDownloader; +import com.yahoo.vespa.filedistribution.FileReferenceCompressor; import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDownload; +import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -39,7 +41,7 @@ public class FileServerTest { @Before public void setup() throws IOException { File rootDir = new File(temporaryFolder.newFolder("fileserver-root").getAbsolutePath()); - fileServer = new FileServer(rootDir, new MockFileDownloader(rootDir)); + fileServer = new FileServer(rootDir, new MockFileDownloader(rootDir), List.of(gzip, lz4)); } @Test @@ -79,11 +81,25 @@ public class FileServerTest { CompletableFuture<byte []> content = new CompletableFuture<>(); fileServer.startFileServing(new FileReference("12y"), new FileReceiver(content), Set.of(gzip)); assertEquals(new String(content.get()), "dummy-data"); + } - IOUtils.writeFile(dir + "/12z/f1", "dummy-data-2", true); - content = new CompletableFuture<>(); - fileServer.startFileServing(new FileReference("12z"), new FileReceiver(content), Set.of(gzip, lz4)); - assertEquals(new String(content.get()), "dummy-data-2"); + @Test + public void requireThatWeCanReplayDirWithLz4() throws IOException, InterruptedException, ExecutionException { + File rootDir = new File(temporaryFolder.newFolder("fileserver-root-3").getAbsolutePath()); + fileServer = new FileServer(rootDir, new MockFileDownloader(rootDir), List.of(lz4, gzip)); // prefer lz4 + File dir = getFileServerRootDir(); + IOUtils.writeFile(dir + "/subdir/12z/f1", "dummy-data-2", true); + CompletableFuture<byte []> content = new CompletableFuture<>(); + fileServer.startFileServing(new FileReference("subdir"), new FileReceiver(content), Set.of(gzip, lz4)); + + // Decompress with lz4 and check contents + var compressor = new FileReferenceCompressor(FileReferenceData.Type.compressed, lz4); + File downloadedFileCompressed = new File(dir + "/downloaded-file-compressed"); + IOUtils.writeFile(downloadedFileCompressed, content.get()); + File downloadedFileUncompressed = new File(dir + "/downloaded-file-uncompressed"); + compressor.decompress(downloadedFileCompressed, downloadedFileUncompressed); + assertTrue(downloadedFileUncompressed.isDirectory()); + assertEquals("dummy-data-2", IOUtils.readFile(new File(downloadedFileUncompressed, "12z/f1"))); } @Test @@ -124,7 +140,7 @@ public class FileServerTest { private FileServer createFileServer(ConfigserverConfig.Builder configBuilder) throws IOException { File fileReferencesDir = temporaryFolder.newFolder(); configBuilder.fileReferencesDir(fileReferencesDir.getAbsolutePath()); - return new FileServer(new ConfigserverConfig(configBuilder)); + return new FileServer(new ConfigserverConfig(configBuilder), new InMemoryFlagSource()); } private static class FileReceiver implements FileServer.Receiver { @@ -149,7 +165,8 @@ public class FileServerTest { new Supervisor(new Transport("mock")).setDropEmptyBuffers(true), downloadDirectory, Duration.ofMillis(100), - Duration.ofMillis(100)); + Duration.ofMillis(100), + Set.of(gzip)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index 7ad237e45ed..8607fc0e2dc 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -5,6 +5,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.component.Version; +import com.yahoo.config.FileReference; import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; @@ -27,16 +28,20 @@ import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.filedistribution.LazyFileReferenceData; import com.yahoo.vespa.model.VespaModel; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.xml.sax.SAXException; - import java.io.File; import java.io.IOException; import java.util.Optional; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; +import static com.yahoo.vespa.config.server.rpc.RpcServer.ChunkedFileReceiver.createMetaRequest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -97,6 +102,25 @@ public class RpcServerTest { } } + @Test + public void testFileReceiverMetaRequest() throws IOException { + File file = temporaryFolder.newFile(); + Request request = createMetaRequest(new LazyFileReferenceData(new FileReference("foo"), "fileA", compressed, file, gzip)); + assertEquals(4, request.parameters().size()); + assertEquals("foo", request.parameters().get(0).asString()); + assertEquals("fileA", request.parameters().get(1).asString()); + assertEquals("compressed", request.parameters().get(2).asString()); + assertEquals(0, request.parameters().get(3).asInt64()); + + request = createMetaRequest(new LazyFileReferenceData(new FileReference("foo"), "fileA", compressed, file, lz4)); + assertEquals(5, request.parameters().size()); + assertEquals("foo", request.parameters().get(0).asString()); + assertEquals("fileA", request.parameters().get(1).asString()); + assertEquals("compressed", request.parameters().get(2).asString()); + assertEquals(0, request.parameters().get(3).asInt64()); + assertEquals("lz4", request.parameters().get(4).asString()); + } + private JRTClientConfigRequest createSimpleRequest() { ConfigKey<?> key = new ConfigKey<>(SimpletypesConfig.class, ""); JRTClientConfigRequest clientReq = createRequest(new RawConfig(key, SimpletypesConfig.getDefMd5())); diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 1378008b546..9c14c03ba21 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -1091,6 +1091,7 @@ "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder enable(boolean)", "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder port(int)", "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder clientTimeout(double)", + "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder handlerTimeout(double)", "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder cacheExpiry(double)", "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy build()" ], @@ -1108,6 +1109,7 @@ "public boolean enable()", "public int port()", "public double clientTimeout()", + "public double handlerTimeout()", "public double cacheExpiry()" ], "fields": [] diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java index b949edefb31..8fa658bf7fc 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java @@ -22,9 +22,9 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import static com.yahoo.container.jdisc.state.JsonUtil.sanitizeDouble; @@ -61,6 +61,7 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { private final Timer timer; private final SnapshotProvider snapshotProvider; private final String applicationName; + private final String hostDimension; @Inject public MetricsPacketsHandler(StateMonitor monitor, @@ -71,6 +72,7 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { this.timer = timer; snapshotProvider = getSnapshotProviderOrThrow(snapshotProviders); applicationName = config.application(); + hostDimension = config.hostname(); } @@ -173,16 +175,16 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { } private void addDimensions(MetricDimensions metricDimensions, ObjectNode packet) { - if (metricDimensions == null) return; - - Iterator<Map.Entry<String, String>> dimensionsIterator = metricDimensions.iterator(); - if (dimensionsIterator.hasNext()) { - ObjectNode jsonDim = jsonMapper.createObjectNode(); - packet.set(DIMENSIONS_KEY, jsonDim); - for (Map.Entry<String, String> dimensionEntry : metricDimensions) { - jsonDim.put(dimensionEntry.getKey(), dimensionEntry.getValue()); - } + if (metricDimensions == null && hostDimension.isEmpty()) return; + + ObjectNode jsonDim = jsonMapper.createObjectNode(); + packet.set(DIMENSIONS_KEY, jsonDim); + Iterable<Map.Entry<String, String>> dimensionIterator = metricDimensions == null ? Set.of() : metricDimensions; + for (Map.Entry<String, String> dimensionEntry : dimensionIterator) { + jsonDim.put(dimensionEntry.getKey(), dimensionEntry.getValue()); } + if (!hostDimension.isEmpty() && !jsonDim.has("host")) + jsonDim.put("host", hostDimension); } private void addMetrics(MetricSet metricSet, ObjectNode packet) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java index fcb243e094d..f559a368fe3 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java @@ -66,8 +66,10 @@ class HealthCheckProxyHandler extends HandlerWrapper { ConnectorConfig.HealthCheckProxy proxyConfig = connector.connectorConfig().healthCheckProxy(); if (proxyConfig.enable()) { Duration targetTimeout = Duration.ofMillis((int) (proxyConfig.clientTimeout() * 1000)); + Duration handlerTimeout = Duration.ofMillis((int) (proxyConfig.handlerTimeout() * 1000)); Duration cacheExpiry = Duration.ofMillis((int) (proxyConfig.cacheExpiry() * 1000)); - ProxyTarget target = createProxyTarget(proxyConfig.port(), targetTimeout, cacheExpiry, connectors); + ProxyTarget target = createProxyTarget( + proxyConfig.port(), targetTimeout, handlerTimeout, cacheExpiry, connectors); mapping.put(connector.listenPort(), target); log.info(String.format("Port %1$d is configured as a health check proxy for port %2$d. " + "HTTP requests to '%3$s' on %1$d are proxied as HTTPS to %2$d.", @@ -77,8 +79,8 @@ class HealthCheckProxyHandler extends HandlerWrapper { return mapping; } - private static ProxyTarget createProxyTarget(int targetPort, Duration targetTimeout, Duration cacheExpiry, - List<JDiscServerConnector> connectors) { + private static ProxyTarget createProxyTarget(int targetPort, Duration clientTimeout, Duration handlerTimeout, + Duration cacheExpiry, List<JDiscServerConnector> connectors) { JDiscServerConnector targetConnector = connectors.stream() .filter(connector -> connector.listenPort() == targetPort) .findAny() @@ -91,7 +93,7 @@ class HealthCheckProxyHandler extends HandlerWrapper { .orElseThrow(() -> new IllegalArgumentException("Health check proxy can only target https port")); ConnectorConfig.ProxyProtocol proxyProtocolCfg = targetConnector.connectorConfig().proxyProtocol(); boolean proxyProtocol = proxyProtocolCfg.enabled() && !proxyProtocolCfg.mixedMode(); - return new ProxyTarget(targetPort, targetTimeout, cacheExpiry, sslContextFactory, proxyProtocol); + return new ProxyTarget(targetPort, clientTimeout,handlerTimeout, cacheExpiry, sslContextFactory, proxyProtocol); } @Override @@ -103,7 +105,7 @@ class HealthCheckProxyHandler extends HandlerWrapper { ServletOutputStream out = servletResponse.getOutputStream(); if (servletRequest.getRequestURI().equals(HEALTH_CHECK_PATH)) { ProxyRequestTask task = new ProxyRequestTask(asyncContext, proxyTarget, servletResponse, out); - asyncContext.setTimeout(proxyTarget.timeout.plusSeconds(1).toMillis()); // add additional time for response sending + asyncContext.setTimeout(proxyTarget.handlerTimeout.toMillis()); asyncContext.addListener(new AsyncListener() { @Override public void onStartAsync(AsyncEvent event) {} @Override public void onComplete(AsyncEvent event) {} @@ -212,20 +214,22 @@ class HealthCheckProxyHandler extends HandlerWrapper { private static class ProxyTarget implements AutoCloseable { final int port; - final Duration timeout; + final Duration clientTimeout; + final Duration handlerTimeout; final Duration cacheExpiry; final SslContextFactory.Server serverSsl; final boolean proxyProtocol; volatile HttpClient client; volatile StatusResponse lastResponse; - ProxyTarget(int port, Duration timeout, Duration cacheExpiry, SslContextFactory.Server serverSsl, - boolean proxyProtocol) { + ProxyTarget(int port, Duration clientTimeout, Duration handlerTimeout, Duration cacheExpiry, + SslContextFactory.Server serverSsl, boolean proxyProtocol) { this.port = port; - this.timeout = timeout; + this.clientTimeout = clientTimeout; this.cacheExpiry = cacheExpiry; this.serverSsl = serverSsl; this.proxyProtocol = proxyProtocol; + this.handlerTimeout = handlerTimeout; } StatusResponse requestStatusHtml() { @@ -239,8 +243,8 @@ class HealthCheckProxyHandler extends HandlerWrapper { private StatusResponse getStatusResponse() { try { var request = client().newRequest("https://localhost:" + port + HEALTH_CHECK_PATH); - request.timeout(timeout.toMillis(), TimeUnit.MILLISECONDS); - request.idleTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS); + request.timeout(clientTimeout.toMillis(), TimeUnit.MILLISECONDS); + request.idleTimeout(clientTimeout.toMillis(), TimeUnit.MILLISECONDS); if (proxyProtocol) { request.tag(new ProxyProtocolClientConnectionFactory.V1.Tag()); } @@ -265,7 +269,7 @@ class HealthCheckProxyHandler extends HandlerWrapper { if (client == null) { synchronized (this) { if (client == null) { - int timeoutMillis = (int) timeout.toMillis(); + int timeoutMillis = (int) clientTimeout.toMillis(); SslContextFactory.Client clientSsl = new SslContextFactory.Client(); clientSsl.setHostnameVerifier((__, ___) -> true); clientSsl.setSslContext(getSslContext(serverSsl)); diff --git a/container-core/src/main/resources/configdefinitions/container.jdisc.state.metrics-packets-handler.def b/container-core/src/main/resources/configdefinitions/container.jdisc.state.metrics-packets-handler.def index 9ec81b7db1b..ab0362c3995 100644 --- a/container-core/src/main/resources/configdefinitions/container.jdisc.state.metrics-packets-handler.def +++ b/container-core/src/main/resources/configdefinitions/container.jdisc.state.metrics-packets-handler.def @@ -4,3 +4,5 @@ namespace=container.jdisc.state # The name of the application that is reporting metrics. application string +# Optional hostname to add as dimension +hostname string default=""
\ No newline at end of file diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def index 2a8e7159b2f..e808b565e8b 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def @@ -101,9 +101,12 @@ healthCheckProxy.enable bool default=false # Which port to proxy healthCheckProxy.port int default=8080 -# Low-level timeout for proxy client (socket connect, socket read, connection pool). Aggregate timeout will be longer. +# Low-level timeout for proxy client (socket connect, socket read, connection pool). healthCheckProxy.clientTimeout double default=1.0 +# Servlet async request timeout. Must be larger than 'clientTimeout' to cover cost of queueing and response handling. +healthCheckProxy.handlerTimeout double default=1.5 + # Expiry for cached health response healthCheckProxy.cacheExpiry double default=1.0 diff --git a/container-core/src/main/sh/find-pid b/container-core/src/main/sh/find-pid index 4db6562db1b..57393140e0b 100755 --- a/container-core/src/main/sh/find-pid +++ b/container-core/src/main/sh/find-pid @@ -74,7 +74,7 @@ findhost # END environment bootstrap section -set -euo pipefail +set -uo pipefail if (( $# != 1 )); then echo "Usage: $0 <service-name-or-config-id>" >&2 diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java index 4d44281658c..f8f567e1890 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java @@ -50,6 +50,7 @@ public class ComponentGraphTest { super(); } + @SuppressWarnings("deprecation") public <T extends ConfigInstance> ConfigMap add(Class<T> clazz, String configId) { ConfigKey<T> key = new ConfigKey<>(clazz, configId); put(key, ConfigGetter.getConfig(key.getConfigClass(), key.getConfigId())); diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java index 29452f7babe..2215fcdf4dd 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java @@ -62,6 +62,7 @@ public class ReuseComponentsTest { SimpleComponent throwsException = getComponent(newGraph, SimpleComponent.class); } + @SuppressWarnings("deprecation") @Test public void require_that_component_is_not_reused_when_config_is_changed() { Class<ComponentTakingConfig> componentClass = ComponentTakingConfig.class; @@ -80,6 +81,7 @@ public class ReuseComponentsTest { assertNotSame(instance2, instance); } + @SuppressWarnings("deprecation") @Test public void require_that_component_is_not_reused_when_injected_component_is_changed() { Function<String, ComponentGraph> buildGraph = config -> { @@ -144,6 +146,7 @@ public class ReuseComponentsTest { assertNotSame(newSimpleComponentRegistry, oldSimpleComponentRegistry); } + @SuppressWarnings("deprecation") @Test public void require_that_injected_component_is_reused_even_when_dependent_component_is_changed() { Function<String, ComponentGraph> buildGraph = config -> { diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java index 99a31640429..6c05af95289 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java @@ -27,13 +27,14 @@ import static org.junit.Assert.assertTrue; public class MetricsPacketsHandlerTest extends StateHandlerTestBase { private static final String APPLICATION_NAME = "state-handler-test-base"; + private static final String HOST_DIMENSION = "some-hostname"; private static MetricsPacketsHandler metricsPacketsHandler; @Before public void setupHandler() { metricsPacketsHandlerConfig = new MetricsPacketsHandlerConfig(new MetricsPacketsHandlerConfig.Builder() - .application(APPLICATION_NAME)); + .application(APPLICATION_NAME).hostname(HOST_DIMENSION)); metricsPacketsHandler = new MetricsPacketsHandler(monitor, timer, snapshotProviderRegistry, metricsPacketsHandlerConfig); testDriver = new RequestHandlerTestDriver(metricsPacketsHandler); } @@ -138,6 +139,26 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { List<JsonNode> packets = incrementTimeAndGetJsonPackets(); assertEquals(3, packets.size()); } + + @Test + public void host_dimension_only_created_if_absent() throws Exception { + var context1 = StateMetricContext.newInstance(Map.of("dim1", "value1", "host", "foo.bar")); + var context2 = StateMetricContext.newInstance(Map.of("dim2", "value2")); + var snapshot = new MetricSnapshot(); + snapshot.add(context1, "counter1", 1); + snapshot.add(context2, "counter2", 2); + snapshotProvider.setSnapshot(snapshot); + + var packets = incrementTimeAndGetJsonPackets(); + assertEquals(3, packets.size()); + + packets.forEach(packet -> { + if (!packet.has(DIMENSIONS_KEY)) return; + var dimensions = packet.get(DIMENSIONS_KEY); + if (dimensions.has("dim1")) assertDimension(packet, "host", "foo.bar"); + if (dimensions.has("dim2")) assertDimension(packet, "host", HOST_DIMENSION); + }); + } private List<JsonNode> incrementTimeAndGetJsonPackets() throws Exception { advanceToNextSnapshot(); @@ -163,6 +184,13 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { assertEquals(expected, counterMetrics.get(metricName).asLong()); } + private void assertDimension(JsonNode metricsPacket, String dimensionName, String expectedDimensionValue) { + assertTrue(metricsPacket.has(DIMENSIONS_KEY)); + var dimensions = metricsPacket.get(DIMENSIONS_KEY); + assertTrue(dimensions.has(dimensionName)); + assertEquals(expectedDimensionValue, dimensions.get(dimensionName).asText()); + } + private void createSnapshotWithCountMetric(String name, Number value, MetricDimensions context) { var snapshot = new MetricSnapshot(); snapshot.add(context, name, value); diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java index e3042310ad0..cf50eeb2c8d 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java @@ -1,12 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.shared; +import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.jdisc.AbstractResource; - -import java.util.Objects; -import java.util.logging.Level; - import com.yahoo.messagebus.DestinationSessionParams; import com.yahoo.messagebus.IntermediateSessionParams; import com.yahoo.messagebus.MessageBus; @@ -15,8 +12,8 @@ import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.rpc.RPCNetwork; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; -import com.yahoo.cloud.config.SlobroksConfig; - +import java.util.Objects; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -57,6 +54,7 @@ public class SharedMessageBus extends AbstractResource { return new SharedMessageBus(new MessageBus(newNetwork(netParams), mbusParams)); } + @SuppressWarnings("deprecation") private static Network newNetwork(RPCNetworkParams params) { SlobroksConfig cfg = params.getSlobroksConfig(); if (cfg == null) { diff --git a/container-messagebus/src/main/resources/configdefinitions/container.jdisc.container-mbus.def b/container-messagebus/src/main/resources/configdefinitions/container.jdisc.container-mbus.def index 143e02a30ef..bd43c13aba3 100644 --- a/container-messagebus/src/main/resources/configdefinitions/container.jdisc.container-mbus.def +++ b/container-messagebus/src/main/resources/configdefinitions/container.jdisc.container-mbus.def @@ -22,10 +22,6 @@ transport_events_before_wakeup int default=1 # Dynamic throttling is used, and works better than anything else. maxpendingcount int default=2048 -enabled bool default=false -#maxpendingsize is set in megabytes! -maxpendingsize int default=100 - #The amount of input data that the service can process concurrently maxConcurrentFactor double default=0.2 range=[0.0-1.0] diff --git a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java index 5b8c99506c5..bf0272f4f66 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java +++ b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java @@ -22,7 +22,7 @@ import java.util.Map; * @author baldersheim */ class Json2SingleLevelMap { - private static final ObjectMapper jsonMapper = new ObjectMapper(); + private static final ObjectMapper jsonMapper = new ObjectMapper().configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); private final byte [] buf; private final JsonParser parser; Json2SingleLevelMap(InputStream data) { diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 76702a1d4e0..3da3f57cb21 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -41,7 +41,6 @@ import com.yahoo.search.searchchain.ExecutionFactory; import com.yahoo.search.searchchain.SearchChainRegistry; import com.yahoo.search.statistics.ElapsedTime; import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; import com.yahoo.yolean.Exceptions; import com.yahoo.yolean.trace.TraceNode; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java index 31278af9579..5b30e3c383d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java @@ -21,6 +21,7 @@ import java.util.Set; */ public class QueryProfileConfigurer { + @SuppressWarnings("deprecation") public static QueryProfileRegistry createFromConfigId(String configId) { return createFromConfig(ConfigGetter.getConfig(QueryProfilesConfig.class, configId)); } diff --git a/container-search/src/test/java/com/yahoo/prelude/IndexFactsFactory.java b/container-search/src/test/java/com/yahoo/prelude/IndexFactsFactory.java index 88cc066b665..482c9e3c6ba 100644 --- a/container-search/src/test/java/com/yahoo/prelude/IndexFactsFactory.java +++ b/container-search/src/test/java/com/yahoo/prelude/IndexFactsFactory.java @@ -23,6 +23,7 @@ public abstract class IndexFactsFactory { } + @SuppressWarnings("deprecation") private static <T extends ConfigInstance> T resolveConfig(Class<T> configClass, String configId) { if (configId == null) return null; return ConfigGetter.getConfig(configClass, configId); diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java index 229416f7a85..42bc1c22529 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java @@ -62,6 +62,7 @@ public class ParsingTester { * Returns an unfrozen version of the IndexFacts this will use. * This can be used to add new indexes and passing the resulting IndexFacts to the constructor of this. */ + @SuppressWarnings("deprecation") public static IndexFacts createIndexFacts() { String indexInfoConfigID = "file:src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg"; ConfigGetter<IndexInfoConfig> getter = new ConfigGetter<>(IndexInfoConfig.class); diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/StemmingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/StemmingSearcherTestCase.java index 55ce7ed4e99..0d7b0caa65b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/StemmingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/StemmingSearcherTestCase.java @@ -96,6 +96,7 @@ public class StemmingSearcherTestCase { "WEAKAND(100) notnoun:tower notnoun:tower notnoun:tow"); } + @SuppressWarnings("deprecation") @Test public void testEmptyIndexInfo() { String indexInfoConfigID = "file:src/test/java/com/yahoo/prelude/querytransform/test/emptyindexinfo.cfg"; diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java index ef3526f2fb1..0b926f5e9b7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java @@ -4,21 +4,18 @@ package com.yahoo.prelude.searcher.test; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.hitfield.HitField; import com.yahoo.prelude.searcher.QrQuotetableConfig; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.Relevance; +import com.yahoo.prelude.searcher.QuotingSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.hitfield.HitField; import com.yahoo.search.Searcher; -import com.yahoo.prelude.searcher.QuotingSearcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import org.junit.Test; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -34,6 +31,7 @@ import static org.junit.Assert.assertTrue; */ public class QuotingSearcherTestCase { + @SuppressWarnings("deprecation") public static QuotingSearcher createQuotingSearcher(String configId) { QrQuotetableConfig config = new ConfigGetter<>(QrQuotetableConfig.class).getConfig(configId); return new QuotingSearcher(new ComponentId("QuotingSearcher"), config); diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java index 84b8ef32871..6b8c268f4ca 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java @@ -2,24 +2,23 @@ package com.yahoo.prelude.searcher.test; import com.yahoo.component.chain.Chain; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.search.Searcher; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.search.config.ClusterConfig; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.searcher.ValidateSortingSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.config.ClusterConfig; +import com.yahoo.search.searchchain.Execution; import com.yahoo.search.test.QueryTestCase; +import com.yahoo.vespa.config.search.AttributesConfig; import org.junit.Test; - import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; /** * Check sorting validation behaves OK. @@ -30,6 +29,7 @@ public class ValidateSortingSearcherTestCase { private final ValidateSortingSearcher searcher; + @SuppressWarnings("deprecation") public ValidateSortingSearcherTestCase() { QrSearchersConfig.Builder qrsCfg = new QrSearchersConfig.Builder(); qrsCfg.searchcluster(new QrSearchersConfig.Searchcluster.Builder().name("giraffes")); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java index 0515417f515..7d9ea07339f 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java @@ -4,18 +4,16 @@ package com.yahoo.prelude.semantics.test; import com.yahoo.component.chain.Chain; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.semantics.SemanticRulesConfig; -import com.yahoo.search.Query; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleBaseException; +import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.prelude.semantics.SemanticSearcher; +import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.test.QueryTestCase; import org.junit.Test; - import java.util.ArrayList; import java.util.List; @@ -28,6 +26,7 @@ import static org.junit.Assert.fail; * * @author bratseth */ +@SuppressWarnings("deprecation") public class ConfigurationTestCase { private static final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java index 0f7c9526533..6a54492f5e5 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java @@ -1,39 +1,29 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.test; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - import com.google.common.collect.ImmutableList; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.container.QrSearchersConfig; -import com.yahoo.search.config.IndexInfoConfig; -import com.yahoo.search.config.IndexInfoConfig.Indexinfo; -import com.yahoo.search.config.IndexInfoConfig.Indexinfo.Alias; -import com.yahoo.search.config.IndexInfoConfig.Indexinfo.Command; import com.yahoo.language.process.StemMode; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.search.Query; +import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.searchchain.Execution; import org.junit.Test; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * Tests using synthetic index names for IndexFacts class. @@ -45,6 +35,7 @@ public class IndexFactsTestCase { private static final String INDEXFACTS_TESTING = "file:src/test/java/com/yahoo/prelude/test/indexfactstesting.cfg"; + @SuppressWarnings("deprecation") private IndexFacts createIndexFacts() { ConfigGetter<IndexInfoConfig> getter = new ConfigGetter<>(IndexInfoConfig.class); IndexInfoConfig config = getter.getConfig(INDEXFACTS_TESTING); diff --git a/container-search/src/test/java/com/yahoo/search/handler/Json2SinglelevelMapTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/Json2SinglelevelMapTestCase.java index d8db88323c7..c50b063ff44 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/Json2SinglelevelMapTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/Json2SinglelevelMapTestCase.java @@ -29,4 +29,10 @@ public class Json2SinglelevelMapTestCase { assertEquals("null", m.get("n")); assertEquals("[0.786, 0.193]", m.get("a")); } + @Test + public void testThatWeAllowSingleQuotes() { + Map<String, String> m = new Json2SingleLevelMap(new ByteArrayInputStream("{'yql':'text'}".getBytes(StandardCharsets.UTF_8))).parse(); + assertTrue(m.containsKey("yql")); + assertEquals("text", m.get("yql")); + } } diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java index da005871539..c12a3fb0e14 100644 --- a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java @@ -40,6 +40,7 @@ public class QueryRewriteSearcherTestUtils { * * @param configPath path for the searcher config */ + @SuppressWarnings("deprecation") public static RewritesConfig createConfigObj(String configPath) { return new ConfigGetter<>(RewritesConfig.class).getConfig(configPath); } diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java index 587b40dfd03..577963ef2b3 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.searchers; -import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; @@ -13,10 +12,9 @@ import com.yahoo.search.query.parser.ParserEnvironment; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.yql.YqlParser; -import com.yahoo.vespa.config.search.AttributesConfig.Attribute; import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.AttributesConfig.Attribute; import org.junit.Test; - import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -33,35 +31,25 @@ public class ValidateFuzzySearcherTestCase { List<String> attributes; public ValidateFuzzySearcherTestCase() { - int i = 0; attributes = new ArrayList<>(); - StringBuilder attributeConfig = new StringBuilder(); + AttributesConfig.Builder configBuilder = new AttributesConfig.Builder(); + List<AttributesConfig.Attribute.Builder> attributesList = new ArrayList<>(); for (Attribute.Datatype.Enum attr: Attribute.Datatype.Enum.values()) { for (Attribute.Collectiontype.Enum ctype: Attribute.Collectiontype.Enum.values()) { + AttributesConfig.Attribute.Builder attributesBuilder = new AttributesConfig.Attribute.Builder(); String attributeName = attr.name().toLowerCase() + "_" + ctype.name().toLowerCase(); + attributesBuilder.name(attributeName); + attributesBuilder.datatype(attr); + attributesBuilder.collectiontype(ctype); + attributesList.add(attributesBuilder); - attributeConfig.append("attribute[" + i + "].name "); - attributeConfig.append(attributeName); - attributeConfig.append("\n"); - - attributeConfig.append("attribute[" + i + "].datatype "); - attributeConfig.append(attr.name()); - attributeConfig.append("\n"); - - attributeConfig.append("attribute[" + i + "].collectiontype "); - attributeConfig.append(ctype.name()); - attributeConfig.append("\n"); - - i += 1; attributes.add(attributeName); } } + configBuilder.attribute(attributesList); + AttributesConfig config = configBuilder.build(); - searcher = new ValidateFuzzySearcher(ConfigGetter.getConfig( - AttributesConfig.class, - "raw: " + - "attribute[" + attributes.size() + "]\n" + - attributeConfig)); + searcher = new ValidateFuzzySearcher(config); } private String makeQuery(String attribute, String query, int maxEditDistance, int prefixLength) { diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java index 9a760bfb0cb..c2f4ee31aa8 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java @@ -2,28 +2,22 @@ package com.yahoo.search.searchers; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.FileSource; -import com.yahoo.config.subscription.RawSource; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.Result; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.yql.YqlParser; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.AttributesConfig; - -import com.yahoo.vespa.config.search.RankProfilesConfig; import org.junit.Test; -import java.io.File; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -34,6 +28,7 @@ public class ValidateNearestNeighborTestCase { ValidateNearestNeighborSearcher searcher; + @SuppressWarnings("deprecation") public ValidateNearestNeighborTestCase() { searcher = new ValidateNearestNeighborSearcher( ConfigGetter.getConfig(AttributesConfig.class, diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java index 01e360858c1..1b3689a94b0 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.search.searchers.test; import com.yahoo.component.chain.Chain; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.RawSource; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchers.ValidateMatchPhaseSearcher; @@ -15,7 +14,7 @@ import org.junit.Test; import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** * @author baldersheim @@ -24,6 +23,7 @@ public class ValidateMatchPhaseSearcherTestCase { private final ValidateMatchPhaseSearcher searcher; + @SuppressWarnings("deprecation") public ValidateMatchPhaseSearcherTestCase() { searcher = new ValidateMatchPhaseSearcher( ConfigGetter.getConfig(AttributesConfig.class, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index a381bcd5bed..3355f765f42 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -41,6 +41,7 @@ public class Node { private final Version wantedVersion; private final Version currentOsVersion; private final Version wantedOsVersion; + private final boolean deferOsUpgrade; private final DockerImage currentDockerImage; private final DockerImage wantedDockerImage; private final ServiceState serviceState; @@ -76,7 +77,7 @@ public class Node { private Node(String id, HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, NodeResources resources, Optional<ApplicationId> owner, Version currentVersion, Version wantedVersion, - Version currentOsVersion, Version wantedOsVersion, Optional<Instant> currentFirmwareCheck, + Version currentOsVersion, Version wantedOsVersion, boolean deferOsUpgrade, Optional<Instant> currentFirmwareCheck, Optional<Instant> wantedFirmwareCheck, ServiceState serviceState, Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, int cost, int failCount, Optional<String> flavor, String clusterId, @@ -97,6 +98,7 @@ public class Node { this.wantedVersion = Objects.requireNonNull(wantedVersion, "wantedVersion must be non-null"); this.currentOsVersion = Objects.requireNonNull(currentOsVersion, "currentOsVersion must be non-null"); this.wantedOsVersion = Objects.requireNonNull(wantedOsVersion, "wantedOsVersion must be non-null"); + this.deferOsUpgrade = deferOsUpgrade; this.currentFirmwareCheck = Objects.requireNonNull(currentFirmwareCheck, "currentFirmwareCheck must be non-null"); this.wantedFirmwareCheck = Objects.requireNonNull(wantedFirmwareCheck, "wantedFirmwareCheck must be non-null"); this.serviceState = Objects.requireNonNull(serviceState, "serviceState must be non-null"); @@ -184,6 +186,11 @@ public class Node { return wantedOsVersion; } + /** Returns whether the node is currently deferring any OS upgrade */ + public boolean deferOsUpgrade() { + return deferOsUpgrade; + } + /** The container image of this is currently running */ public DockerImage currentDockerImage() { return currentDockerImage; @@ -455,6 +462,7 @@ public class Node { private Version wantedVersion = Version.emptyVersion; private Version currentOsVersion = Version.emptyVersion; private Version wantedOsVersion = Version.emptyVersion; + private boolean deferOsUpgrade = false; private DockerImage currentDockerImage = DockerImage.EMPTY; private DockerImage wantedDockerImage = DockerImage.EMPTY; private Optional<Instant> currentFirmwareCheck = Optional.empty(); @@ -502,6 +510,7 @@ public class Node { this.wantedVersion = node.wantedVersion; this.currentOsVersion = node.currentOsVersion; this.wantedOsVersion = node.wantedOsVersion; + this.deferOsUpgrade = node.deferOsUpgrade; this.currentDockerImage = node.currentDockerImage; this.wantedDockerImage = node.wantedDockerImage; this.serviceState = node.serviceState; @@ -599,6 +608,11 @@ public class Node { return this; } + public Builder deferOsUpgrade(boolean deferOsUpgrade) { + this.deferOsUpgrade = deferOsUpgrade; + return this; + } + public Builder currentDockerImage(DockerImage currentDockerImage) { this.currentDockerImage = currentDockerImage; return this; @@ -761,7 +775,7 @@ public class Node { public Node build() { return new Node(id, hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, - currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState, + currentOsVersion, wantedOsVersion, deferOsUpgrade, currentFirmwareCheck, wantedFirmwareCheck, serviceState, suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration, cost, failCount, flavor, clusterId, clusterType, group, index, retired, wantToRetire, wantToDeprovision, wantToRebuild, down, reservedTo, exclusiveTo, wantedDockerImage, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 09ddcf4fa5e..b6905a97b5f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -161,7 +161,7 @@ public final class JobType implements Comparable<JobType> { /** A serialized form of this: {@code <environment>.<region>[.test]}; the inverse of {@link #ofSerialized(String)} */ public String serialized() { - return zone.environment().value() + "." + zone.region().value() + (isProductionTest ? ".test" : ""); + return zone().value() + (isProductionTest ? ".test" : ""); } public String jobName() { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TestReport.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TestReport.java index 789a909b780..60cb4248035 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TestReport.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TestReport.java @@ -1,10 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; -import com.yahoo.slime.SlimeUtils; - -import java.nio.charset.StandardCharsets; - /** * @author mortent */ @@ -21,7 +17,6 @@ public class TestReport { } public static TestReport fromJson(String report) { - SlimeUtils.jsonToSlimeOrThrow(report.getBytes(StandardCharsets.UTF_8)); // Verify structure. return new TestReport(report); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java index 8964e1b5127..d8a804ecd2c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java @@ -63,6 +63,8 @@ public class NodeRepositoryNode { private String currentOsVersion; @JsonProperty("wantedOsVersion") private String wantedOsVersion; + @JsonProperty("deferOsUpgrade") + private Boolean deferOsUpgrade; @JsonProperty("currentFirmwareCheck") private Long currentFirmwareCheck; @JsonProperty("wantedFirmwareCheck") @@ -250,6 +252,14 @@ public class NodeRepositoryNode { this.wantedVespaVersion = wantedVespaVersion; } + public Boolean getDeferOsUpgrade() { + return deferOsUpgrade; + } + + public void setDeferOsUpgrade(Boolean deferOsUpgrade) { + this.deferOsUpgrade = deferOsUpgrade; + } + public Integer getFailCount() { return failCount; } @@ -441,12 +451,13 @@ public class NodeRepositoryNode { // --- end + @Override public String toString() { return "NodeRepositoryNode{" + "url='" + url + '\'' + ", id='" + id + '\'' + - ", state=" + state + + ", state='" + state + '\'' + ", hostname='" + hostname + '\'' + ", ipAddresses=" + ipAddresses + ", additionalIpAddresses=" + additionalIpAddresses + @@ -464,11 +475,12 @@ public class NodeRepositoryNode { ", wantedVespaVersion='" + wantedVespaVersion + '\'' + ", currentOsVersion='" + currentOsVersion + '\'' + ", wantedOsVersion='" + wantedOsVersion + '\'' + + ", deferOsUpgrade=" + deferOsUpgrade + ", currentFirmwareCheck=" + currentFirmwareCheck + ", wantedFirmwareCheck=" + wantedFirmwareCheck + ", failCount=" + failCount + - ", environment=" + environment + - ", type=" + type + + ", environment='" + environment + '\'' + + ", type='" + type + '\'' + ", wantedDockerImage='" + wantedDockerImage + '\'' + ", currentDockerImage='" + currentDockerImage + '\'' + ", parentHostname='" + parentHostname + '\'' + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 045e43f532c..b9432fdc375 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -237,7 +237,7 @@ public class ApplicationController { } /** Sets the default target major version. Set to empty to determine target version normally (by confidence) */ - public void setTargetMajorVersion(Optional<Integer> targetMajorVersion) { + public void setTargetMajorVersion(OptionalInt targetMajorVersion) { curator.writeTargetMajorVersion(targetMajorVersion); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 7ceeda08d3a..d83f552ab25 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -228,10 +228,9 @@ public class DeploymentTrigger { Instance instance = application.require(applicationId.instance()); JobId job = new JobId(instance.id(), jobType); JobStatus jobStatus = jobs.jobStatus(new JobId(applicationId, jobType)); - Versions versions = jobStatus.lastTriggered() - .orElseThrow(() -> new IllegalArgumentException(job + " has never been triggered")) - .versions(); - trigger(deploymentJob(instance, versions, jobType, jobStatus, clock.instant()), reason); + Run last = jobStatus.lastTriggered() + .orElseThrow(() -> new IllegalArgumentException(job + " has never been triggered")); + trigger(deploymentJob(instance, last.versions(), last.id().type(), jobStatus.isNodeAllocationFailure(), clock.instant()), reason); return job; } @@ -259,7 +258,12 @@ public class DeploymentTrigger { .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); jobs.forEach((jobId, versionsList) -> { - trigger(deploymentJob(application.require(job.application().instance()), versionsList.get(0).versions(), jobId.type(), status.jobs().get(jobId).get(), clock.instant()), reason); + trigger(deploymentJob(application.require(job.application().instance()), + versionsList.get(0).versions(), + jobId.type(), + status.jobs().get(jobId).get().isNodeAllocationFailure(), + clock.instant()), + reason); }); return List.copyOf(jobs.keySet()); } @@ -388,7 +392,7 @@ public class DeploymentTrigger { jobs.add(deploymentJob(status.application().require(jobId.application().instance()), job.versions(), job.type(), - status.instanceJobs(jobId.application().instance()).get(jobId.type()), + status.instanceJobs(jobId.application().instance()).get(jobId.type()).isNodeAllocationFailure(), job.readyAt().get())); }); return Collections.unmodifiableList(jobs); @@ -475,8 +479,8 @@ public class DeploymentTrigger { // ---------- Version and job helpers ---------- - private Job deploymentJob(Instance instance, Versions versions, JobType jobType, JobStatus jobStatus, Instant availableSince) { - return new Job(instance, versions, jobType, availableSince, jobStatus.isNodeAllocationFailure(), instance.change().revision().isPresent()); + private Job deploymentJob(Instance instance, Versions versions, JobType jobType, boolean isNodeAllocationFailure, Instant availableSince) { + return new Job(instance, versions, jobType, availableSince, isNodeAllocationFailure, instance.change().revision().isPresent()); } // ---------- Data containers ---------- diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 19b2afb3af9..881107fa0f9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -158,7 +158,7 @@ public class JobController { /** Stores the given log entries for the given run and step. */ public void log(RunId id, Step step, List<LogEntry> entries) { locked(id, __ -> { - logs.append(id.application(), id.type(), step, entries); + logs.append(id.application(), id.type(), step, entries, true); return __; }); } @@ -211,7 +211,7 @@ public class JobController { if (log.isEmpty()) return run; - logs.append(id.application(), id.type(), Step.copyVespaLogs, log); + logs.append(id.application(), id.type(), Step.copyVespaLogs, log, false); return run.with(log.get(log.size() - 1).at()); }); } @@ -230,7 +230,7 @@ public class JobController { if (entries.isEmpty()) return run; - logs.append(id.application(), id.type(), step.get(), entries); + logs.append(id.application(), id.type(), step.get(), entries, false); return run.with(entries.stream().mapToLong(LogEntry::id).max().getAsLong()); }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java index 09e0fec41d1..c8c5a1834c7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java @@ -14,14 +14,17 @@ public class ApplicationMetaDataGarbageCollector extends ControllerMaintainer { private static final Logger log = Logger.getLogger(ApplicationMetaDataGarbageCollector.class.getName()); + private final Duration timeToLive; + public ApplicationMetaDataGarbageCollector(Controller controller, Duration interval) { super(controller, interval); + this.timeToLive = controller.system().isCd() ? Duration.ofDays(7) : Duration.ofDays(365); } @Override protected double maintain() { try { - controller().applications().applicationStore().pruneMeta(controller().clock().instant().minus(Duration.ofDays(365))); + controller().applications().applicationStore().pruneMeta(controller().clock().instant().minus(timeToLive)); return 1.0; } catch (Exception e) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 4aeecdcd4ff..9793cded918 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -70,7 +70,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new ArchiveAccessMaintainer(controller, metric, intervals.archiveAccessMaintainer)); maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer)); maintainers.add(new ChangeRequestMaintainer(controller, intervals.changeRequestMaintainer)); - maintainers.add(new VcmrMaintainer(controller, intervals.vcmrMaintainer)); + maintainers.add(new VcmrMaintainer(controller, intervals.vcmrMaintainer, metric)); maintainers.add(new CloudTrialExpirer(controller, intervals.defaultInterval)); maintainers.add(new RetriggerMaintainer(controller, intervals.retriggerMaintainer)); maintainers.add(new UserManagementMaintainer(controller, intervals.userManagementMaintainer, controller.serviceRegistry().roleMaintainer())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index 3bd1c7bb358..111931b638b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -9,8 +9,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepo import com.yahoo.vespa.hosted.controller.api.integration.deployment.OsRelease; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; +import java.time.DayOfWeek; import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.Objects; @@ -84,14 +86,11 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } /** OS release based on a tag */ - private static class TaggedRelease implements Release { + private record TaggedRelease(SystemName system, ArtifactRepository artifactRepository) implements Release { - private final SystemName system; - private final ArtifactRepository artifactRepository; - - private TaggedRelease(SystemName system, ArtifactRepository artifactRepository) { - this.system = Objects.requireNonNull(system); - this.artifactRepository = Objects.requireNonNull(artifactRepository); + public TaggedRelease { + Objects.requireNonNull(system); + Objects.requireNonNull(artifactRepository); } @Override @@ -119,41 +118,30 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } /** OS release based on calendar-versioning */ - private static class CalendarVersionedRelease implements Release { + record CalendarVersionedRelease(SystemName system) implements Release { - /** The time to wait before scheduling upgrade to next version */ - private static final Duration SCHEDULING_INTERVAL = Duration.ofDays(45); + /** A fixed point in time which the release schedule is calculated from */ + private static final Instant START_OF_SCHEDULE = LocalDate.of(2022, 1, 1) + .atStartOfDay() + .toInstant(ZoneOffset.UTC); - /** - * The interval at which new versions become available. We use this to avoid scheduling upgrades to a version - * that has not been released yet. Example: Version N is the latest one and target is set to N+1. If N+1 does - * not exist the zone will not converge until N+1 has been released and we may end up triggering multiple - * rounds of upgrades. - */ - private static final Duration AVAILABILITY_INTERVAL = Duration.ofDays(7); + /** The time that should elapse between versions */ + private static final Duration SCHEDULING_STEP = Duration.ofDays(60); - private static final DateTimeFormatter CALENDAR_VERSION_PATTERN = DateTimeFormatter.ofPattern("yyyyMMdd"); + /** The day of week new releases are published */ + private static final DayOfWeek RELEASE_DAY = DayOfWeek.MONDAY; - private final SystemName system; + private static final DateTimeFormatter CALENDAR_VERSION_PATTERN = DateTimeFormatter.ofPattern("yyyyMMdd"); - public CalendarVersionedRelease(SystemName system) { - this.system = Objects.requireNonNull(system); + public CalendarVersionedRelease { + Objects.requireNonNull(system); } @Override public Version version(OsVersionTarget currentTarget, Instant now) { - Instant scheduledAt = currentTarget.scheduledAt(); Version currentVersion = currentTarget.osVersion().version(); - if (scheduledAt.isBefore(now.minus(SCHEDULING_INTERVAL))) { - String calendarVersion = now.minus(AVAILABILITY_INTERVAL) - .atZone(ZoneOffset.UTC) - .format(CALENDAR_VERSION_PATTERN); - return new Version(currentVersion.getMajor(), - currentVersion.getMinor(), - currentVersion.getMicro(), - calendarVersion); - } - return currentVersion; // New version should not be scheduled yet + Version wantedVersion = asVersion(dateOfWantedVersion(now), currentVersion); + return wantedVersion.isAfter(currentVersion) ? wantedVersion : currentVersion; } @Override @@ -161,6 +149,32 @@ public class OsUpgradeScheduler extends ControllerMaintainer { return system.isCd() ? Duration.ZERO : Duration.ofDays(14); } + /** + * Calculate the date of the wanted version relative to now. A given zone will choose the oldest release + * available which is not older than this date. + */ + static LocalDate dateOfWantedVersion(Instant now) { + Instant candidate = START_OF_SCHEDULE; + while (!candidate.plus(SCHEDULING_STEP).isAfter(now)) { + candidate = candidate.plus(SCHEDULING_STEP); + } + LocalDate date = LocalDate.ofInstant(candidate, ZoneOffset.UTC); + return releaseDayOf(date); + } + + private static LocalDate releaseDayOf(LocalDate date) { + int releaseDayDelta = RELEASE_DAY.getValue() - date.getDayOfWeek().getValue(); + return date.plusDays(releaseDayDelta); + } + + private static Version asVersion(LocalDate dateOfVersion, Version currentVersion) { + String calendarVersion = dateOfVersion.format(CALENDAR_VERSION_PATTERN); + return new Version(currentVersion.getMajor(), + currentVersion.getMinor(), + currentVersion.getMicro(), + calendarVersion); + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index fa64a2677f4..8155476f139 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -62,7 +62,7 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { protected boolean expectUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { return cloud.equals(zone.getCloudName()) && // Cloud is managed by this upgrader application.shouldUpgradeOs() && // Application should upgrade in this cloud - canUpgrade(node); // Node is in an upgradable state + canUpgrade(node); } @Override @@ -98,14 +98,12 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { /** Returns whether to spend upgrade budget on given zone */ private boolean spendBudgetOn(ZoneApi zone) { - if (!zone.getEnvironment().isProduction()) return false; - if (controller().zoneRegistry().systemZone().getVirtualId().equals(zone.getVirtualId())) return false; // Controller zone - return true; + return !controller().zoneRegistry().systemZone().getVirtualId().equals(zone.getVirtualId()); // Do not spend budget on controller zone } - /** Returns whether node is in a state where it can be upgraded */ + /** Returns whether node currently allows upgrades */ public static boolean canUpgrade(Node node) { - return upgradableNodeStates.contains(node.state()); + return !node.deferOsUpgrade() && upgradableNodeStates.contains(node.state()); } private static String name(CloudName cloud) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index a2fb0df626f..1932dc65657 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.OptionalInt; import java.util.Random; import java.util.Set; @@ -182,7 +181,7 @@ public class Upgrader extends ControllerMaintainer { } /** Sets the default target major version. Set to empty to determine target version normally (by confidence) */ - public void setTargetMajorVersion(Optional<Integer> targetMajorVersion) { + public void setTargetMajorVersion(OptionalInt targetMajorVersion) { controller().applications().setTargetMajorVersion(targetMajorVersion); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index 551f803f368..daba7e74f34 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.jdisc.Metric; import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -46,26 +47,28 @@ public class VcmrMaintainer extends ControllerMaintainer { private static final Logger LOG = Logger.getLogger(VcmrMaintainer.class.getName()); private static final int DAYS_TO_RETIRE = 2; private static final Duration ALLOWED_POSTPONEMENT_TIME = Duration.ofDays(7); + protected static final String TRACKED_CMRS_METRIC = "cmr.tracked"; private final CuratorDb curator; private final NodeRepository nodeRepository; private final ChangeRequestClient changeRequestClient; private final SystemName system; + private final Metric metric; - public VcmrMaintainer(Controller controller, Duration interval) { + public VcmrMaintainer(Controller controller, Duration interval, Metric metric) { super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.curator = controller.curator(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); this.changeRequestClient = controller.serviceRegistry().changeRequestClient(); this.system = controller.system(); + this.metric = metric; } @Override protected double maintain() { var changeRequests = curator.readChangeRequests() .stream() - .filter(shouldUpdate()) - .collect(Collectors.toList()); + .filter(shouldUpdate()).toList(); var nodesByZone = nodesByZone(); @@ -86,6 +89,7 @@ public class VcmrMaintainer extends ControllerMaintainer { }); } }); + updateMetrics(); return 1.0; } @@ -357,4 +361,15 @@ public class VcmrMaintainer extends ControllerMaintainer { return time; } + private void updateMetrics() { + var cmrsByStatus = curator.readChangeRequests() + .stream() + .collect(Collectors.groupingBy(VespaChangeRequest::getStatus)); + + for (var status : Status.values()) { + var count = cmrsByStatus.getOrDefault(status, List.of()).size(); + metric.set(TRACKED_CMRS_METRIC, count, metric.createContext(Map.of("status", status.name()))); + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java index 9721026c628..ecb9db8195f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java @@ -49,7 +49,7 @@ public class BufferedLogStore { } /** Appends to the log of the given, active run, reassigning IDs as counted here, and converting to Vespa log levels. */ - public void append(ApplicationId id, JobType type, Step step, List<LogEntry> entries) { + public void append(ApplicationId id, JobType type, Step step, List<LogEntry> entries, boolean forceLog) { if (entries.isEmpty()) return; @@ -58,7 +58,7 @@ public class BufferedLogStore { long lastEntryId = buffer.readLastLogEntryId(id, type).orElse(-1L); long lastChunkId = buffer.getLogChunkIds(id, type).max().orElse(0); long numberOfChunks = Math.max(1, buffer.getLogChunkIds(id, type).count()); - if (numberOfChunks > maxLogSize / chunkSize) + if (numberOfChunks > maxLogSize / chunkSize && ! forceLog) return; // Max size exceeded — store no more. byte[] emptyChunk = "[]".getBytes(); @@ -72,8 +72,12 @@ public class BufferedLogStore { buffer.writeLastLogEntryId(id, type, lastEntryId); buffer.writeLog(id, type, lastChunkId, logSerializer.toJson(log)); lastChunkId = lastEntryId + 1; - if (++numberOfChunks > maxLogSize / chunkSize) { - log = Map.of(step, List.of(new LogEntry(++lastEntryId, entry.at(), LogEntry.Type.warning, "Max log size of " + (maxLogSize >> 20) + "Mb exceeded; further entries are discarded."))); + if (++numberOfChunks > maxLogSize / chunkSize && ! forceLog) { + log = Map.of(step, List.of(new LogEntry(++lastEntryId, + entry.at(), + LogEntry.Type.warning, + "Max log size of " + (maxLogSize >> 20) + + "Mb exceeded; further user entries are discarded."))); break; } log = new HashMap<>(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index f02f49e7114..54e98877ba3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.yahoo.component.annotation.Inject; import com.yahoo.collections.Pair; import com.yahoo.component.Version; +import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -41,6 +41,7 @@ import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; + import java.io.IOException; import java.io.UncheckedIOException; import java.nio.ByteBuffer; @@ -53,6 +54,7 @@ import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeoutException; @@ -271,9 +273,9 @@ public class CuratorDb { return read(targetMajorVersionPath(), ByteBuffer::wrap).map(ByteBuffer::getInt); } - public void writeTargetMajorVersion(Optional<Integer> targetMajorVersion) { + public void writeTargetMajorVersion(OptionalInt targetMajorVersion) { if (targetMajorVersion.isPresent()) - curator.set(targetMajorVersionPath(), ByteBuffer.allocate(Integer.BYTES).putInt(targetMajorVersion.get()).array()); + curator.set(targetMajorVersionPath(), ByteBuffer.allocate(Integer.BYTES).putInt(targetMajorVersion.getAsInt()).array()); else curator.delete(targetMajorVersionPath()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index cfb00db7b63..56eaf2f3a2e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -75,6 +75,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; @@ -1473,6 +1474,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse trigger(ApplicationId id, JobType type, HttpRequest request) { + // JobType.fromJobName doesn't properly initiate test jobs. Triggering these without context isn't _really_ + // necessary, but triggering a test in the default cloud is better than failing with a weird error. + ZoneRegistry zones = controller.zoneRegistry(); + type = switch (type.environment()) { + case test -> JobType.systemTest(zones, zones.systemZone().getCloudName()); + case staging -> JobType.stagingTest(zones, zones.systemZone().getCloudName()); + default -> type; + }; + Inspector requestObject = toSlime(request.getData()).get(); boolean requireTests = ! requestObject.field("skipTests").asBool(); boolean reTrigger = requestObject.field("reTrigger").asBool(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 9c016eccd27..25953c16bf0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -115,6 +115,7 @@ class JobControllerApiHandlerHelper { Run run = jobController.run(runId); detailsObject.setBool("active", ! run.hasEnded()); detailsObject.setString("status", nameOf(run.status())); + run.reason().ifPresent(reason -> detailsObject.setString("reason", reason)); try { jobController.updateTestLog(runId); jobController.updateVespaLog(runId); @@ -421,6 +422,7 @@ class JobControllerApiHandlerHelper { runObject.setLong("start", run.start().toEpochMilli()); run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli())); runObject.setString("status", run.status().name()); + run.reason().ifPresent(reason -> runObject.setString("reason", reason)); toSlime(runObject.setObject("versions"), run.versions(), application); Cursor runStepsArray = runObject.setArray("steps"); run.steps().forEach((step, info) -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 25ac90ac0ea..776fcbfd03b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -35,6 +35,7 @@ import java.security.Principal; import java.security.cert.X509Certificate; import java.time.Instant; import java.util.Optional; +import java.util.OptionalInt; import java.util.Scanner; import java.util.function.Function; import java.util.logging.Level; @@ -60,13 +61,13 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { @Override public HttpResponse auditAndHandle(HttpRequest request) { try { - switch (request.getMethod()) { - case GET: return get(request); - case POST: return post(request); - case DELETE: return delete(request); - case PATCH: return patch(request); - default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - } + return switch (request.getMethod()) { + case GET -> get(request); + case POST -> post(request); + case DELETE -> delete(request); + case PATCH -> patch(request); + default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + }; } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); @@ -165,8 +166,8 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { if (inspect.field(upgradesPerMinuteField).valid()) { upgrader.setUpgradesPerMinute(inspect.field(upgradesPerMinuteField).asDouble()); } else if (inspect.field(targetMajorVersionField).valid()) { - int target = (int)inspect.field(targetMajorVersionField).asLong(); - upgrader.setTargetMajorVersion(Optional.ofNullable(target == 0 ? null : target)); // 0 is the default value + int target = (int) inspect.field(targetMajorVersionField).asLong(); + upgrader.setTargetMajorVersion(target == 0 ? OptionalInt.empty() : OptionalInt.of(target)); // 0 is the default value } else { return ErrorResponse.badRequest("No such modifiable field(s)"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index 7f33f612cd0..e078df0267f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -15,7 +15,8 @@ import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; /** * Information about a particular Vespa version. - * VespaVersions are identified by their version number and ordered by increasing version numbers. + * + * Vespa versions are identified by their version number and ordered by increasing version numbers. * * @author bratseth */ @@ -29,8 +30,11 @@ public record VespaVersion(Version version, Confidence confidence) implements Comparable<VespaVersion> { public static Confidence confidenceFrom(DeploymentStatistics statistics, Controller controller) { + int thisMajorVersion = statistics.version().getMajor(); + int defaultMajorVersion = controller.applications().targetMajorVersion().orElse(thisMajorVersion); InstanceList all = InstanceList.from(controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()) - .withProductionDeployment())); + .withProductionDeployment())) + .allowingMajorVersion(thisMajorVersion, defaultMajorVersion); // 'production on this': All production deployment jobs upgrading to this version have completed without failure InstanceList productionOnThis = all.matching(instance -> statistics.productionSuccesses().stream().anyMatch(run -> run.id().application().equals(instance))) .not().failingUpgrade() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 157f6b27487..a8c18957773 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -1153,7 +1153,7 @@ public class ControllerTest { assertEquals(version2, tester.applications().compileVersion(application, OptionalInt.of(8))); // Default major version is set to 8. - tester.applications().setTargetMajorVersion(Optional.of(8)); + tester.applications().setTargetMajorVersion(OptionalInt.of(8)); assertEquals(version1, tester.applications().compileVersion(application, OptionalInt.of(7))); assertEquals(version2, tester.applications().compileVersion(application, OptionalInt.empty())); @@ -1169,13 +1169,13 @@ public class ControllerTest { // Application upgrades to major 8; only major version from deployment spec should cause a downgrade. context.submit(new ApplicationPackageBuilder().region("us-west-1").compileVersion(version2).build()).deploy(); - tester.applications().setTargetMajorVersion(Optional.empty()); + tester.applications().setTargetMajorVersion(OptionalInt.empty()); tester.applications().lockApplicationOrThrow(application, locked -> tester.applications().store(locked.withMajorVersion(null))); assertEquals(version1, tester.applications().compileVersion(application, OptionalInt.of(7))); assertEquals(version2, tester.applications().compileVersion(application, OptionalInt.empty())); // Default major version across all apps should not cause a downgrade. - tester.applications().setTargetMajorVersion(Optional.of(7)); + tester.applications().setTargetMajorVersion(OptionalInt.of(7)); assertEquals(version1, tester.applications().compileVersion(application, OptionalInt.of(7))); assertEquals(version2, tester.applications().compileVersion(application, OptionalInt.empty())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 673cbf9708b..08fc6df37fb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -2153,6 +2153,7 @@ public class DeploymentTriggerTest { Version version2 = new Version("8"); tester.controllerTester().flagSource().withListFlag(PermanentFlags.INCOMPATIBLE_VERSIONS.id(), List.of("8"), String.class); + // App deploys on version1. tester.controllerTester().upgradeSystem(version1); DeploymentContext app = tester.newDeploymentContext() .submit(new ApplicationPackageBuilder().region("us-east-3") @@ -2160,10 +2161,12 @@ public class DeploymentTriggerTest { .build()) .deploy(); + // System upgrades to version2, but the app is not upgraded. tester.controllerTester().upgradeSystem(version2); tester.upgrader().run(); assertEquals(Change.empty(), app.instance().change()); + // App compiles against version2, and upgrades. app.submit(new ApplicationPackageBuilder().region("us-east-3") .compileVersion(version2) .build()); @@ -2171,6 +2174,18 @@ public class DeploymentTriggerTest { assertEquals(version2, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); assertEquals(version2, app.application().revisions().get(tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); + + // App specifies version1 in deployment spec, compiles against version1, pins to version1, and then downgrades. + app.submit(new ApplicationPackageBuilder().region("us-east-3") + .majorVersion(7) + .compileVersion(version1) + .build()); + tester.deploymentTrigger().forceChange(app.instanceId(), app.instance().change().withPin()); + app.deploy(); + assertEquals(version1, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); + assertEquals(version1, app.application().revisions().get(tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); + + // A new app, compiled against version1, is deployed on version1. DeploymentContext newApp = tester.newDeploymentContext("new", "app", "default") .submit(new ApplicationPackageBuilder().region("us-east-3") .compileVersion(version1) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java index 300aa86b5ea..621e847c0d3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -13,7 +13,10 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -27,7 +30,7 @@ public class OsUpgradeSchedulerTest { public void schedule_calendar_versioned_release() { ControllerTester tester = new ControllerTester(); OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); - Instant t0 = Instant.parse("2021-01-23T00:00:00.00Z"); // Outside trigger period + Instant t0 = Instant.parse("2022-01-16T00:00:00.00Z"); // Outside trigger period tester.clock().setInstant(t0); CloudName cloud = CloudName.from("cloud"); @@ -38,8 +41,8 @@ public class OsUpgradeSchedulerTest { scheduler.maintain(); assertTrue("No target set", tester.controller().osVersionTarget(cloud).isEmpty()); - // Target is set - Version version0 = Version.fromString("7.0.0.20210123190005"); + // Target is set manually + Version version0 = Version.fromString("7.0.0.20220101"); tester.controller().upgradeOsIn(cloud, version0, Duration.ofDays(1), false); // Target remains unchanged as it hasn't expired yet @@ -49,9 +52,9 @@ public class OsUpgradeSchedulerTest { assertEquals(version0, tester.controller().osVersionTarget(cloud).get().osVersion().version()); } - // Just over 45 days pass, and a new target replaces the expired one - Version version1 = Version.fromString("7.0.0.20210302"); - tester.clock().advance(Duration.ofDays(15).plus(Duration.ofSeconds(1))); + // Enough days pass that the next release is triggered + Version version1 = Version.fromString("7.0.0.20220228"); + tester.clock().advance(Duration.ofDays(30)); scheduler.maintain(); assertEquals("Target is unchanged because we're outside trigger period", version0, tester.controller().osVersionTarget(cloud).get().osVersion().version()); @@ -60,7 +63,7 @@ public class OsUpgradeSchedulerTest { assertEquals("New target set", version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); - // A few days pass and target remains unchanged + // A few more days pass and target remains unchanged tester.clock().advance(Duration.ofDays(2)); scheduler.maintain(); assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); @@ -112,6 +115,24 @@ public class OsUpgradeSchedulerTest { scheduleUpgradeAfter(Duration.ofDays(1), version1, tester); } + @Test + public void schedule_of_calender_versioned_releases() { + Map<String, String> tests = Map.of("2022-01-01", "2021-12-27", + "2022-03-01", "2021-12-27", + "2022-03-02", "2022-02-28", + "2022-04-30", "2022-02-28", + "2022-05-01", "2022-04-25", + "2022-06-29", "2022-04-25", + "2022-07-01", "2022-06-27", + "2022-08-28", "2022-06-27", + "2022-08-29", "2022-08-29"); + tests.forEach((now, expectedVersion) -> { + Instant instant = LocalDate.parse(now).atStartOfDay().toInstant(ZoneOffset.UTC); + LocalDate dateOfWantedVersion = OsUpgradeScheduler.CalendarVersionedRelease.dateOfWantedVersion(instant); + assertEquals("scheduled wanted version at " + now, LocalDate.parse(expectedVersion), dateOfWantedVersion); + }); + } + private void scheduleUpgradeAfter(Duration duration, Version version, ControllerTester tester) { tester.clock().advance(duration); new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)).maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 3c3f0053e91..3a5b4a90baa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -19,7 +19,9 @@ import java.time.Duration; import java.util.Collection; import java.util.List; import java.util.function.Function; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -52,18 +54,19 @@ public class OsUpgraderTest { OsUpgrader osUpgrader = osUpgrader(upgradePolicy, cloud1, false); // Bootstrap system - List<ZoneId> nonControllerZones = List.of(zone1, zone2, zone3, zone4, zone5).stream() - .map(ZoneApi::getVirtualId) - .collect(Collectors.toList()); + List<ZoneId> nonControllerZones = Stream.of(zone1, zone2, zone3, zone4, zone5) + .map(ZoneApi::getVirtualId) + .collect(Collectors.toList()); tester.configServer().bootstrap(nonControllerZones, List.of(SystemApplication.tenantHost)); tester.configServer().addNodes(List.of(zone0.getVirtualId()), List.of(SystemApplication.controllerHost)); // Add system application that exists in a real system, but isn't eligible for OS upgrades tester.configServer().addNodes(nonControllerZones, List.of(SystemApplication.configServer)); - // Fail a few nodes. Failed nodes should not affect versions + // Change state of a few nodes. These should not affect convergence failNodeIn(zone1, SystemApplication.tenantHost); failNodeIn(zone3, SystemApplication.tenantHost); + Node nodeDeferringOsUpgrade = deferOsUpgradeIn(zone2, SystemApplication.tenantHost); // New OS version released Version version1 = Version.fromString("7.1"); @@ -91,7 +94,7 @@ public class OsUpgraderTest { completeUpgrade(version1, SystemApplication.tenantHost, zone1); statusUpdater.maintain(); assertEquals(5, nodesOn(version1).size()); - assertEquals(11, nodesOn(Version.emptyVersion).size()); + assertEquals(10, nodesOn(Version.emptyVersion).size()); // zone 2 and 3: begins upgrading osUpgrader.maintain(); @@ -102,6 +105,10 @@ public class OsUpgraderTest { // zone 2 and 3: completes upgrade completeUpgrade(version1, SystemApplication.tenantHost, zone2, zone3); + assertEquals("Current version is unchanged for node deferring OS upgrade", Version.emptyVersion, + nodeRepository().list(zone2.getVirtualId(), NodeFilter.all().hostnames(nodeDeferringOsUpgrade.hostname())) + .get(0) + .currentOsVersion()); // zone 4: begins upgrading osUpgrader.maintain(); @@ -158,15 +165,15 @@ public class OsUpgraderTest { // First zone upgrades osUpgrader.maintain(); for (var nodeType : nodeTypes) { - assertEquals("Dev zone gets a zero budget", Duration.ZERO, upgradeBudget(zone1, nodeType, version)); + assertEquals(Duration.ofHours(4), upgradeBudget(zone1, nodeType, version)); completeUpgrade(version, nodeType, zone1); } // Next set of zones upgrade osUpgrader.maintain(); - for (var zone : List.of(zone2, zone3)) { + for (var zone : List.of(zone1, zone2, zone3)) { for (var nodeType : nodeTypes) { - assertEquals("Parallel prod zones share the budget of a single zone", Duration.ofHours(6), + assertEquals("Parallel prod zones share the budget of a single zone", Duration.ofHours(4), upgradeBudget(zone, nodeType, version)); completeUpgrade(version, nodeType, zone); } @@ -175,7 +182,7 @@ public class OsUpgraderTest { // Last zone upgrades osUpgrader.maintain(); for (var nodeType : nodeTypes) { - assertEquals(nodeType + " in last prod zone gets the budget of a single zone", Duration.ofHours(6), + assertEquals(nodeType + " in last prod zone gets the budget of a single zone", Duration.ofHours(4), upgradeBudget(zone4, nodeType, version)); completeUpgrade(version, nodeType, zone4); } @@ -271,13 +278,23 @@ public class OsUpgraderTest { .collect(Collectors.toList()); } - private void failNodeIn(ZoneApi zone, SystemApplication application) { + private Node failNodeIn(ZoneApi zone, SystemApplication application) { + return patchOneNodeIn(zone, application, (node) -> Node.builder(node).state(Node.State.failed).build()); + } + + private Node deferOsUpgradeIn(ZoneApi zone, SystemApplication application) { + return patchOneNodeIn(zone, application, (node) -> Node.builder(node).deferOsUpgrade(true).build()); + } + + private Node patchOneNodeIn(ZoneApi zone, SystemApplication application, UnaryOperator<Node> patcher) { List<Node> nodes = nodeRepository().list(zone.getVirtualId(), NodeFilter.all().applications(application.id())); if (nodes.isEmpty()) { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); - nodeRepository().putNodes(zone.getVirtualId(), Node.builder(node).state(Node.State.failed).build()); + Node newNode = patcher.apply(node); + nodeRepository().putNodes(zone.getVirtualId(), newNode); + return newNode; } /** Simulate OS upgrade of nodes allocated to application. In a real system this is done by the node itself */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 5968b490c09..f053ab7a4a6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -36,7 +36,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.pro import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL; -import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.APPLICATION; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PIN; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; import static org.junit.Assert.assertEquals; @@ -746,7 +745,7 @@ public class UpgraderTest { var default1 = tester.newDeploymentContext("tenant1", "default1", "default").submit(DeploymentContext.applicationPackage()).deploy(); // New major version is released, but we don't want to upgrade to it yet - tester.upgrader().setTargetMajorVersion(Optional.of(6)); + tester.upgrader().setTargetMajorVersion(OptionalInt.of(6)); version = Version.fromString("7.0"); tester.controllerTester().upgradeSystem(version); assertEquals(version, tester.controller().readVersionStatus().systemVersion().get().versionNumber()); @@ -771,7 +770,7 @@ public class UpgraderTest { assertEquals(0, tester.jobs().active().size()); // Now we want to upgrade the latest application - tester.upgrader().setTargetMajorVersion(Optional.empty()); + tester.upgrader().setTargetMajorVersion(OptionalInt.empty()); tester.upgrader().maintain(); tester.triggerJobs(); assertEquals(2, tester.jobs().active().size()); @@ -959,7 +958,7 @@ public class UpgraderTest { tester.controllerTester().upgradeSystem(version0); // Apps target 6 by default - tester.upgrader().setTargetMajorVersion(Optional.of(6)); + tester.upgrader().setTargetMajorVersion(OptionalInt.of(6)); // All applications deploy on current version var app1 = createAndDeploy("app1", "default"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java index bfbd3836ce7..321ec3ad8ea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction.State; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VcmrReport; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest.Status; +import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import org.junit.Before; import org.junit.Test; @@ -25,6 +26,7 @@ import java.time.ZonedDateTime; import java.time.temporal.TemporalAdjusters; import java.util.List; +import static com.yahoo.vespa.hosted.controller.maintenance.VcmrMaintainer.TRACKED_CMRS_METRIC; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -37,6 +39,7 @@ public class VcmrMaintainerTest { private ControllerTester tester; private VcmrMaintainer maintainer; private NodeRepositoryMock nodeRepo; + private MetricsMock metrics; private final ZoneId zoneId = ZoneId.from("prod.us-east-3"); private final ZoneId zone2 = ZoneId.from("prod.us-west-1"); private final HostName host1 = HostName.of("host1"); @@ -47,7 +50,8 @@ public class VcmrMaintainerTest { @Before public void setup() { tester = new ControllerTester(); - maintainer = new VcmrMaintainer(tester.controller(), Duration.ofMinutes(1)); + metrics = new MetricsMock(); + maintainer = new VcmrMaintainer(tester.controller(), Duration.ofMinutes(1), metrics); nodeRepo = tester.serviceRegistry().configServer().nodeRepository().allowPatching(true); } @@ -244,6 +248,8 @@ public class VcmrMaintainerTest { assertEquals(State.OUT_OF_SYNC, action.getState()); assertEquals(Status.OUT_OF_SYNC, writtenChangeRequest.getStatus()); + assertEquals(1, metrics.getMetric(context -> "OUT_OF_SYNC".equals(context.get("status")), TRACKED_CMRS_METRIC).get()); + assertEquals(0, metrics.getMetric(context -> "REQUIRES_OPERATOR_ACTION".equals(context.get("status")), TRACKED_CMRS_METRIC).get()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java index 0f7f97d333a..87280c0c1a3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java @@ -50,19 +50,19 @@ public class BufferedLogStoreTest { assertEquals(Optional.empty(), logs.readFinished(id, -1)); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), -1)); - logs.append(id.application(), id.type(), Step.deployReal, List.of(entry)); + logs.append(id.application(), id.type(), Step.deployReal, List.of(entry), false); assertEquals(List.of(entry0), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), 0)); - logs.append(id.application(), id.type(), Step.deployReal, List.of(entry)); + logs.append(id.application(), id.type(), Step.deployReal, List.of(entry), false); assertEquals(List.of(entry0, entry1), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); assertEquals(List.of(entry1), logs.readActive(id.application(), id.type(), 0).get(Step.deployReal)); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), 1)); - logs.append(id.application(), id.type(), Step.deployReal, List.of(entry, entry, entry)); + logs.append(id.application(), id.type(), Step.deployReal, List.of(entry, entry, entry), false); assertEquals(List.of(entry0, entry1, entry2, entry3, entry4), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); assertEquals(List.of(entry1, entry2, entry3, entry4), @@ -105,17 +105,28 @@ public class BufferedLogStoreTest { logged.remove(logged.size() - 1); logged.remove(logged.size() - 1); logged.remove(logged.size() - 1); - logged.add(new LogEntry(2 * maxChunks, entry.at(), LogEntry.Type.warning, "Max log size of " + ((chunkSize * maxChunks) >> 20) + "Mb exceeded; further entries are discarded.")); + logged.add(new LogEntry(2 * maxChunks, entry.at(), LogEntry.Type.warning, "Max log size of " + ((chunkSize * maxChunks) >> 20) + "Mb exceeded; further user entries are discarded.")); - logs.append(id.application(), id.type(), Step.deployReal, monsterLog); + logs.append(id.application(), id.type(), Step.deployReal, monsterLog, false); assertEquals(logged.size(), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal).size()); assertEquals(logged, logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); + // An additional, forced entry is appended. + LogEntry forced = new LogEntry(logged.size(), entry.at(), entry.type(), entry.message()); + logs.append(id.application(), id.type(), Step.deployReal, List.of(forced), true); + logged.add(forced); + assertEquals(logged.size(), + logs.readActive(id.application(), id.type(), -1).get(Step.deployReal).size()); + assertEquals(logged, + logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); + logged.remove(logged.size() - 1); + + // Flushing the buffer clears it again, and makes it ready for reuse. logs.flush(id); for (int i = 0; i < 2 * maxChunks + 3; i++) - logs.append(id.application(), id.type(), Step.deployReal, List.of(entry)); + logs.append(id.application(), id.type(), Step.deployReal, List.of(entry), false); assertEquals(logged.size(), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal).size()); assertEquals(logged, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 696f1ef0ba3..38e9d8c823e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -428,6 +428,7 @@ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/run/1", "start": 1600000000000, "status": "running", + "reason": "triggered by user.myuser", "versions": { "targetPlatform": "6.1.0", "targetApplication": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 12430b67539..2477e8df56e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -9,6 +9,7 @@ "start": 1600000000000, "end": 1600000000000, "status": "success", + "reason": "triggered by user.myuser", "versions": { "targetPlatform": "6.1.0", "targetApplication": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 7a137d4e410..7b045e508a9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -27,6 +28,7 @@ import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.Map; +import java.util.OptionalInt; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -218,7 +220,7 @@ public class VersionStatusTest { tester.upgrader().maintain(); // Setup applications - all running on version0 - ApplicationPackage canaryPolicy = applicationPackage("canary"); + ApplicationPackage canaryPolicy = applicationPackage("canary", 7); var canary0 = tester.newDeploymentContext("tenant1", "canary0", "default") .submit(canaryPolicy) .deploy(); @@ -231,7 +233,7 @@ public class VersionStatusTest { ApplicationPackage defaultPolicy = applicationPackage("default"); var default0 = tester.newDeploymentContext("tenant1", "default0", "default") - .submit(defaultPolicy) + .submit(applicationPackage("default", 7)) .deploy(); var default1 = tester.newDeploymentContext("tenant1", "default1", "default") .submit(defaultPolicy) @@ -379,6 +381,28 @@ public class VersionStatusTest { assertTrue(versions.get(0).isReleased()); assertFalse(versions.get(1).isReleased()); // tesst quirk: maven repo lost during controller recreation; useful to test status though assertTrue(versions.get(2).isReleased()); + + // A new major version is released and all canaries upgrade + Version version4 = new Version("7.1"); + tester.controller().applications().setTargetMajorVersion(OptionalInt.of(version3.getMajor())); // Previous remains the default + tester.controllerTester().upgradeSystem(version4); + tester.upgrader().maintain(); + tester.triggerJobs(); + canary0.deployPlatform(version4); + canary1.deployPlatform(version4); + canary2.deployPlatform(version4); + tester.controllerTester().computeVersionStatus(); + assertEquals(Confidence.normal, confidence(tester.controller(), version4)); + + // The single application allowing this major upgrades and confidence becomes 'high' + tester.upgrader().maintain(); + tester.triggerJobs(); + assertEquals(Change.of(version4), default0.instance().change()); + default0.jobAborted(systemTest) + .jobAborted(stagingTest) + .deployPlatform(version4); + tester.controllerTester().computeVersionStatus(); + assertEquals(Confidence.high, confidence(tester.controller(), version4)); } @Test @@ -687,6 +711,14 @@ public class VersionStatusTest { .orElseThrow(() -> new IllegalArgumentException("Expected to find version: " + version)); } + private static ApplicationPackage applicationPackage(String upgradePolicy, int majorVersion) { + return new ApplicationPackageBuilder().upgradePolicy(upgradePolicy) + .region("us-west-1") + .region("us-east-3") + .majorVersion(majorVersion) + .build(); + } + private static final ApplicationPackage canaryApplicationPackage = new ApplicationPackageBuilder().upgradePolicy("canary") .region("us-west-1") @@ -707,12 +739,12 @@ public class VersionStatusTest { /** Returns empty prebuilt applications for efficiency */ private ApplicationPackage applicationPackage(String upgradePolicy) { - switch (upgradePolicy) { - case "canary" : return canaryApplicationPackage; - case "default" : return defaultApplicationPackage; - case "conservative" : return conservativeApplicationPackage; - default : throw new IllegalArgumentException("No upgrade policy '" + upgradePolicy + "'"); - } + return switch (upgradePolicy) { + case "canary" -> canaryApplicationPackage; + case "default" -> defaultApplicationPackage; + case "conservative" -> conservativeApplicationPackage; + default -> throw new IllegalArgumentException("No upgrade policy '" + upgradePolicy + "'"); + }; } } diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java index 76f4578ac87..c6ba63b7924 100644 --- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java +++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java @@ -125,6 +125,7 @@ public class IndexingProcessorTestCase { return lst.get(0); } + @SuppressWarnings("deprecation") private static IndexingProcessor newProcessor(String configId) { return new IndexingProcessor(new DocumentTypeManager(ConfigGetter.getConfig(DocumentmanagerConfig.class, configId)), ConfigGetter.getConfig(IlscriptsConfig.class, configId), diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def index b1929e42d34..0d0f3876f15 100644 --- a/document/src/vespa/document/config/documentmanager.def +++ b/document/src/vespa/document/config/documentmanager.def @@ -5,9 +5,6 @@ namespace=document.config ## Whether attempts to set an undefined field should be ignored rather than causing an error ignoreundefinedfields bool default=false -## Whether to enable compression in this process. -enablecompression bool default=false - ## Prefer "Vespa 8" format for the "position" type usev8geopositions bool default=false diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def index 3138e71e025..0c135db7b0d 100644 --- a/document/src/vespa/document/config/documenttypes.def +++ b/document/src/vespa/document/config/documenttypes.def @@ -5,9 +5,6 @@ namespace=document.config ## Whether attempts to set an undefined field should be ignored rather than causing an error ignoreundefinedfields bool default=false -## Whether to enable compression in this process. -enablecompression bool default=false - ## Prefer "Vespa 8" format for the "position" type usev8geopositions bool default=false diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index 3607f652a51..1c729008e2c 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -230,7 +230,6 @@ public class MessageBusVisitorSession implements VisitorSession { @Override public Sender createSender(ReplyHandler replyHandler, VisitorParameters visitorParameters) { - messageBus.setMaxPendingCount(0); SourceSessionParams sessionParams = createSourceSessionParams(visitorParameters); return new MessageBusSender(messageBus.createSourceSession(replyHandler, sessionParams)); } @@ -308,7 +307,7 @@ public class MessageBusVisitorSession implements VisitorSession { private static final Logger log = Logger.getLogger(MessageBusVisitorSession.class.getName()); - private static AtomicLong sessionCounter = new AtomicLong(0); + private static final AtomicLong sessionCounter = new AtomicLong(0); private static long getNextSessionId() { return sessionCounter.incrementAndGet(); } @@ -336,7 +335,7 @@ public class MessageBusVisitorSession implements VisitorSession { private boolean done = false; private boolean destroying = false; // For testing and sanity checking private final Object completionMonitor = new Object(); - private Trace trace; + private final Trace trace; /** * We keep our own track of pending messages since the sender's pending * count cannot be relied on in an async task execution context. This diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java index 8b3bc32ff71..ea8461b42f3 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java @@ -12,7 +12,7 @@ public class EmptyFileReferenceData extends FileReferenceData { private int contentRead = 0; private EmptyFileReferenceData(FileReference fileReference, String filename, Type type, byte[] content, long xxhash) { - super(fileReference, filename, type); + super(fileReference, filename, type, CompressionType.gzip); this.content = content; this.xxhash = xxhash; } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 5941ed536a8..63ae8faacfe 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -6,11 +6,11 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.defaults.Defaults; - import java.io.File; import java.time.Duration; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -19,6 +19,8 @@ import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; + /** * Handles downloads of files (file references only for now) * @@ -29,6 +31,7 @@ public class FileDownloader implements AutoCloseable { private static final Logger log = Logger.getLogger(FileDownloader.class.getName()); private static final Duration defaultSleepBetweenRetries = Duration.ofSeconds(5); public static final File defaultDownloadDirectory = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution")); + private static final boolean forceDownload = Boolean.parseBoolean(System.getenv("VESPA_CONFIG_PROXY_FORCE_DOWNLOAD_OF_FILE_REFERENCES")); private final ConnectionPool connectionPool; private final Supervisor supervisor; @@ -37,19 +40,20 @@ public class FileDownloader implements AutoCloseable { private final FileReferenceDownloader fileReferenceDownloader; private final Downloads downloads = new Downloads(); - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout) { - this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout, Set<CompressionType> acceptedCompressionTypes) { + this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries, acceptedCompressionTypes); } - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout) { - this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout, Set<CompressionType> acceptedCompressionTypes) { + this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries, acceptedCompressionTypes); } public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout, - Duration sleepBetweenRetries) { + Duration sleepBetweenRetries, + Set<CompressionType> acceptedCompressionTypes) { this.connectionPool = connectionPool; this.supervisor = supervisor; this.downloadDirectory = downloadDirectory; @@ -60,7 +64,10 @@ public class FileDownloader implements AutoCloseable { downloads, timeout, sleepBetweenRetries, - downloadDirectory); + downloadDirectory, + acceptedCompressionTypes); + if (forceDownload) + log.log(Level.INFO, "Force download of file references (download even if file reference exists on disk)"); } public Optional<File> getFile(FileReferenceDownload fileReferenceDownload) { @@ -99,6 +106,8 @@ public class FileDownloader implements AutoCloseable { } private static Optional<File> getFileFromFileSystem(FileReference fileReference, File downloadDirectory) { + if (forceDownload) return Optional.empty(); + File[] files = new File(downloadDirectory, fileReference.value()).listFiles(); if (files == null) return Optional.empty(); if (files.length == 0) return Optional.empty(); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 65c6dd5931d..a285fbaafe2 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -129,6 +129,7 @@ public class FileReceiver { moveFileToDestination(inprogressFile, file); } else { decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile(); + log.log(Level.FINE, () -> "compression type to use=" + compressionType); new FileReferenceCompressor(fileType, compressionType).decompress(inprogressFile, decompressedDir); moveFileToDestination(decompressedDir, fileReferenceDir); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java index b485e6ded86..efb845bafe7 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java @@ -118,9 +118,9 @@ public class FileReferenceCompressor { } private OutputStream compressedOutputStream(File outputFile) throws IOException { - log.log(Level.FINE, () -> "Compressing with type " + type + " and compression type " + compressionType); switch (type) { case compressed: + log.log(Level.FINE, () -> "Compressing with compression type " + compressionType); switch (compressionType) { case gzip: return new GZIPOutputStream(new FileOutputStream(outputFile)); @@ -137,9 +137,9 @@ public class FileReferenceCompressor { } private InputStream decompressedInputStream(File inputFile) throws IOException { - log.log(Level.FINE, () -> "Decompressing with type " + type + " and compression type " + compressionType); switch (type) { case compressed: + log.log(Level.FINE, () -> "Decompressing with compression type " + compressionType); switch (compressionType) { case gzip: return new GZIPInputStream(new FileInputStream(inputFile)); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java index d14f690b2d3..3f83cbea506 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java @@ -18,11 +18,13 @@ public abstract class FileReferenceData { private final FileReference fileReference; private final String filename; private final Type type; + private final CompressionType compressionType; - public FileReferenceData(FileReference fileReference, String filename, Type type) { + public FileReferenceData(FileReference fileReference, String filename, Type type, CompressionType compressionType) { this.fileReference = fileReference; this.filename = filename; this.type = type; + this.compressionType = compressionType; } public FileReference fileReference() {return fileReference;} @@ -31,6 +33,8 @@ public abstract class FileReferenceData { public Type type() {return type;} + public CompressionType compressionType() { return compressionType;} + public ByteBuffer content() { ByteBuffer bb = ByteBuffer.allocate((int)size()); while (bb.remaining() > 0) { diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java index 8d6f428eaef..f3a8cf9299d 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java @@ -18,7 +18,7 @@ public class FileReferenceDownload { private final boolean downloadFromOtherSourceIfNotFound; private final String client; - public FileReferenceDownload(FileReference fileReference, String client) { + public FileReferenceDownload(FileReference fileReference, String client) { this(fileReference, client, true); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 0267feb9ffc..7078c5aae6c 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -5,14 +5,16 @@ import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.FileReference; import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Request; +import com.yahoo.jrt.StringArray; import com.yahoo.jrt.StringValue; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; - import java.io.File; import java.time.Duration; import java.time.Instant; +import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -20,8 +22,10 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; + /** - * Downloads file reference using rpc requests to config server and keeps track of files being downloaded + * Downloads file reference from config server and keeps track of files being downloaded * * @author hmusum */ @@ -38,12 +42,14 @@ public class FileReferenceDownloader { private final Duration sleepBetweenRetries; private final Duration rpcTimeout; private final File downloadDirectory; + private final Set<CompressionType> acceptedCompressionTypes; FileReferenceDownloader(ConnectionPool connectionPool, Downloads downloads, Duration timeout, Duration sleepBetweenRetries, - File downloadDirectory) { + File downloadDirectory, + Set<CompressionType> acceptedCompressionTypes) { this.connectionPool = connectionPool; this.downloads = downloads; this.downloadTimeout = timeout; @@ -51,6 +57,7 @@ public class FileReferenceDownloader { this.downloadDirectory = downloadDirectory; String timeoutString = System.getenv("VESPA_CONFIGPROXY_FILEDOWNLOAD_RPC_TIMEOUT"); this.rpcTimeout = Duration.ofSeconds(timeoutString == null ? 30 : Integer.parseInt(timeoutString)); + this.acceptedCompressionTypes = requireNonEmpty(acceptedCompressionTypes); } private void waitUntilDownloadStarted(FileReferenceDownload fileReferenceDownload) { @@ -132,6 +139,9 @@ public class FileReferenceDownloader { Request request = new Request("filedistribution.serveFile"); request.parameters().add(new StringValue(fileReferenceDownload.fileReference().value())); request.parameters().add(new Int32Value(fileReferenceDownload.downloadFromOtherSourceIfNotFound() ? 0 : 1)); + String[] temp = new String[acceptedCompressionTypes.size()]; + acceptedCompressionTypes.stream().map(Enum::name).toList().toArray(temp); + request.parameters().add(new StringArray(temp)); return request; } @@ -160,4 +170,9 @@ public class FileReferenceDownloader { } } + private static Set<CompressionType> requireNonEmpty(Set<CompressionType> s) { + if (Objects.requireNonNull(s).isEmpty()) throw new IllegalArgumentException("set must be non-empty"); + return s; + } + } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java index 10de9c072b9..c8fbb639b35 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java @@ -17,8 +17,8 @@ public class LazyFileReferenceData extends FileReferenceData { private final ReadableByteChannel channel; private final StreamingXXHash64 hasher; - public LazyFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException { - super(fileReference, filename, type); + public LazyFileReferenceData(FileReference fileReference, String filename, Type type, File file, CompressionType compressionType) throws IOException { + super(fileReference, filename, type, compressionType); this.file = file; channel = Files.newByteChannel(file.toPath()); this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java index 974d5ff1489..a69370b28ff 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java @@ -12,8 +12,8 @@ import java.nio.file.Files; */ public class LazyTemporaryStorageFileReferenceData extends LazyFileReferenceData { - public LazyTemporaryStorageFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException { - super(fileReference, filename, type, file); + public LazyTemporaryStorageFileReferenceData(FileReference fileReference, String filename, Type type, File file, CompressionType compressionType) throws IOException { + super(fileReference, filename, type, file, compressionType); } public void close() { diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index f5cd1760e89..629ea5915df 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -17,7 +17,6 @@ import net.jpountz.xxhash.XXHashFactory; import org.junit.After; import org.junit.Before; import org.junit.Test; - import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; @@ -26,6 +25,7 @@ import java.nio.file.Path; import java.time.Duration; import java.util.Arrays; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -41,6 +41,7 @@ import static org.junit.Assert.fail; public class FileDownloaderTest { private static final Duration sleepBetweenRetries = Duration.ofMillis(10); + private static final Set<FileReferenceData.CompressionType> acceptedCompressionTypes = Set.of(gzip); private MockConnection connection; private FileDownloader fileDownloader; @@ -53,7 +54,7 @@ public class FileDownloaderTest { downloadDir = Files.createTempDirectory("filedistribution").toFile(); connection = new MockConnection(); supervisor = new Supervisor(new Transport()).setDropEmptyBuffers(true); - fileDownloader = new FileDownloader(connection, supervisor, downloadDir, Duration.ofSeconds(1), sleepBetweenRetries); + fileDownloader = createDownloader(connection, Duration.ofSeconds(1)); } catch (IOException e) { e.printStackTrace(); fail(e.getMessage()); @@ -167,7 +168,7 @@ public class FileDownloaderTest { @Test public void getFileWhenConnectionError() throws IOException { - fileDownloader = new FileDownloader(connection, supervisor, downloadDir, Duration.ofSeconds(2), sleepBetweenRetries); + fileDownloader = createDownloader(connection, Duration.ofSeconds(2)); File downloadDir = fileDownloader.downloadDirectory(); int timesToFail = 2; @@ -201,7 +202,7 @@ public class FileDownloaderTest { public void getFileWhenDownloadInProgress() throws IOException, ExecutionException, InterruptedException { ExecutorService executor = Executors.newFixedThreadPool(2); String filename = "abc.jar"; - fileDownloader = new FileDownloader(connection, supervisor, downloadDir, Duration.ofSeconds(3), sleepBetweenRetries); + fileDownloader = createDownloader(connection, Duration.ofSeconds(3)); File downloadDir = fileDownloader.downloadDirectory(); // Delay response so that we can make a second request while downloading the file from the first request @@ -241,7 +242,7 @@ public class FileDownloaderTest { Duration timeout = Duration.ofMillis(200); MockConnection connectionPool = new MockConnection(); connectionPool.setResponseHandler(new MockConnection.WaitResponseHandler(timeout.plus(Duration.ofMillis(1000)))); - FileDownloader fileDownloader = new FileDownloader(connectionPool, supervisor, downloadDir, timeout, sleepBetweenRetries); + FileDownloader fileDownloader = createDownloader(connectionPool, timeout); FileReference xyzzy = new FileReference("xyzzy"); // Should download since we do not have the file on disk fileDownloader.downloadIfNeeded(new FileReferenceDownload(xyzzy, "test")); @@ -264,6 +265,16 @@ public class FileDownloaderTest { assertEquals("content", IOUtils.readFile(downloadedFile)); } + @Test + public void testCompressionTypes() { + try { + createDownloader(connection, Duration.ofSeconds(1), Set.of()); + fail("expected to fail when set is empty"); + } catch (IllegalArgumentException e) { + // ignore + } + } + private void writeFileReference(File dir, String fileReferenceString, String fileName) throws IOException { File fileReferenceDir = new File(dir, fileReferenceString); fileReferenceDir.mkdir(); @@ -302,6 +313,14 @@ public class FileDownloaderTest { return fileDownloader.getFile(new FileReferenceDownload(fileReference, "test")); } + private FileDownloader createDownloader(MockConnection connection, Duration timeout) { + return createDownloader(connection, timeout, acceptedCompressionTypes); + } + + private FileDownloader createDownloader(MockConnection connection, Duration timeout, Set<FileReferenceData.CompressionType> acceptedCompressionTypes) { + return new FileDownloader(connection, supervisor, downloadDir, timeout, sleepBetweenRetries, acceptedCompressionTypes); + } + private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { private ResponseHandler responseHandler; diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java index 66b731f204b..eda93331c73 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java @@ -7,12 +7,15 @@ import com.yahoo.text.Utf8; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -27,7 +30,7 @@ public class FileReferenceDataTest { String content = "blob"; File tempFile = writeTempFile(content); FileReferenceData fileReferenceData = - new LazyTemporaryStorageFileReferenceData(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, tempFile); + new LazyTemporaryStorageFileReferenceData(new FileReference("ref"), "foo", compressed, tempFile, gzip); ByteBuffer byteBuffer = ByteBuffer.allocate(100); assertEquals(4, fileReferenceData.nextContent(byteBuffer)); assertEquals(content, Utf8.toString(Arrays.copyOfRange(byteBuffer.array(), 0, 4))); @@ -44,7 +47,7 @@ public class FileReferenceDataTest { String content = "blobbblubbblabb"; File file = writeTempFile(content); FileReferenceData fileReferenceData = - new LazyFileReferenceData(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, file); + new LazyFileReferenceData(new FileReference("ref"), "foo", Type.compressed, file, CompressionType.gzip); ByteBuffer byteBuffer = ByteBuffer.allocate(10); assertEquals(10, fileReferenceData.nextContent(byteBuffer)); assertEquals(content.substring(0,10), Utf8.toString(Arrays.copyOfRange(byteBuffer.array(), 0, 10))); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 78b7f5fff76..bbe11e18d93 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -68,6 +68,12 @@ public class Flags { "Takes effect at redeployment (requires restart)", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag KEEP_STORAGE_NODE_UP = defineFeatureFlag( + "keep-storage-node-up", true, + List.of("hakonhall"), "2022-07-07", "2022-08-07", + "Whether to leave the storage node (with wanted state) UP while the node is permanently down.", + "Takes effect immediately for nodes transitioning to permanently down.", + ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_UNCOMMITTED_MEMORY = defineIntFlag( "max-uncommitted-memory", 130000, @@ -153,6 +159,43 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag MBUS_JAVA_NUM_TARGETS = defineIntFlag( + "mbus-java-num-targets", 1, + List.of("baldersheim"), "2022-07-05", "2023-01-01", + "Number of rpc targets per service", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag MBUS_CPP_NUM_TARGETS = defineIntFlag( + "mbus-cpp-num-targets", 1, + List.of("baldersheim"), "2022-07-05", "2023-01-01", + "Number of rpc targets per service", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag RPC_NUM_TARGETS = defineIntFlag( + "rpc-num-targets", 1, + List.of("baldersheim"), "2022-07-05", "2023-01-01", + "Number of rpc targets per content node", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag MBUS_JAVA_EVENTS_BEFORE_WAKEUP = defineIntFlag( + "mbus-java-events-before-wakeup", 1, + List.of("baldersheim"), "2022-07-05", "2023-01-01", + "Number write events before waking up transport thread", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag MBUS_CPP_EVENTS_BEFORE_WAKEUP = defineIntFlag( + "mbus-cpp-events-before-wakeup", 1, + List.of("baldersheim"), "2022-07-05", "2023-01-01", + "Number write events before waking up transport thread", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag RPC_EVENTS_BEFORE_WAKEUP = defineIntFlag( + "rpc-events-before-wakeup", 1, + List.of("baldersheim"), "2022-07-05", "2023-01-01", + "Number write events before waking up transport thread", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag MBUS_NUM_THREADS = defineIntFlag( "mbus-num-threads", 4, List.of("baldersheim"), "2022-07-01", "2023-01-01", @@ -366,9 +409,9 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag APPLICATION_FILES_WITH_UNKNOWN_EXTENSION = defineStringFlag( - "fail-deployment-for-files-with-unknown-extension", "NOOP", + "fail-deployment-for-files-with-unknown-extension", "LOG", List.of("hmusum"), "2022-04-27", "2022-07-27", - "Whether to log, fail or do nothing for deployments when app has a file with unknown extension (valid values LOG, FAIL, NOOP)", + "Whether to log, fail or do nothing for deployments when app has a file with unknown extension (valid values LOG, FAIL, NOOP)", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -393,13 +436,6 @@ public class Flags { "Takes effect on redeployment", APPLICATION_ID); - public static final UnboundStringFlag FILE_DISTRIBUTION_COMPRESSION_ALGORITHM = defineStringFlag( - "file-distribution-compression-algorithm", "gzip", - List.of("hmusum"), "2022-05-24", "2022-07-24", - "Which algorithm to use for compressing file references when distributing files. Valid values: none, gzip", - "Takes effect immediately", - APPLICATION_ID); - public static final UnboundListFlag<String> FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES = defineListFlag( "file-distribution-accepted-compression-types", List.of("gzip"), String.class, List.of("hmusum"), "2022-07-05", "2022-09-05", @@ -429,12 +465,19 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag FIX_IPV6_GATEWAY = defineFeatureFlag( - "fix-ipv6-gateway", false, + "fix-ipv6-gateway", true, List.of("mpolden"), "2022-07-04", "2022-09-01", "Fix a misconfigured IPv6 gateway automatically", "Takes effect on first host admin resume", HOSTNAME); + public static final UnboundBooleanFlag SEPARATE_METRIC_CHECK_CONFIG = defineFeatureFlag( + "separate-metric-check-config", false, + List.of("olaa"), "2022-07-04", "2022-09-01", + "Determines whether one metrics config check should be written per Vespa node", + "Takes effect on next tick", + HOSTNAME); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 3bd5d5d4890..19d3b5b3e43 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -470,6 +470,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @return The resender. */ + @Deprecated // Remove on 9 public Resender getResender() { return resender; } @@ -480,6 +481,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @return The pending count. */ + @Deprecated // Package private on 9 public synchronized int getPendingCount() { return pendingCount; } @@ -490,6 +492,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @return The pending size. */ + @Deprecated // Package private on 9 public synchronized int getPendingSize() { return pendingSize; } @@ -500,6 +503,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @param maxCount The max count. */ + @Deprecated // Remove on 9 public void setMaxPendingCount(int maxCount) { maxPendingCount = maxCount; } @@ -508,6 +512,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * Gets maximum number of messages that can be received without being * replied to yet. */ + @Deprecated // Remove on 9 public int getMaxPendingCount() { return maxPendingCount; } @@ -518,6 +523,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @param maxSize The max size. */ + @Deprecated // Remove on 9 public void setMaxPendingSize(int maxSize) { maxPendingSize = maxSize; } @@ -526,6 +532,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * Gets maximum combined size of messages that can be received without * being replied to yet. */ + @Deprecated // Remove on 9 public int getMaxPendingSize() { return maxPendingSize; } diff --git a/messagebus/src/test/java/com/yahoo/messagebus/ChokeTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/ChokeTestCase.java index 60ec6c400a9..6d50fc769be 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/ChokeTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/ChokeTestCase.java @@ -14,7 +14,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import static org.junit.Assert.assertEquals; @@ -55,6 +54,7 @@ public class ChokeTestCase { } @Test + @SuppressWarnings("deprecation") public void testMaxCount() { int max = 10; dstServer.mb.setMaxPendingCount(max); @@ -103,6 +103,7 @@ public class ChokeTestCase { } @Test + @SuppressWarnings("deprecation") public void testMaxSize() { int size = createMessage("msg").getApproxSize(); int max = size * 10; diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index 8f9037f70dd..412dc29d4f2 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -17,7 +17,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/util/gate.h> #include <vespa/fastos/thread.h> #include <thread> @@ -136,12 +136,9 @@ RPCNetwork::RPCNetwork(const RPCNetworkParams ¶ms) : _targetPool(std::make_unique<RPCTargetPool>(params.getConnectionExpireSecs(), params.getNumRpcTargets())), _targetPoolTask(std::make_unique<TargetPoolTask>(_scheduler, *_targetPool)), _servicePool(std::make_unique<RPCServicePool>(*_mirror, 4_Ki)), - _executor(std::make_unique<vespalib::ThreadStackExecutor>(params.getNumThreads(), 64_Ki)), _sendV2(std::make_unique<RPCSendV2>()), _sendAdapters(), - _compressionConfig(params.getCompressionConfig()), - _allowDispatchForEncode(params.getDispatchOnEncode()), - _allowDispatchForDecode(params.getDispatchOnDecode()) + _compressionConfig(params.getCompressionConfig()) { } @@ -413,7 +410,6 @@ void RPCNetwork::sync() { SyncTask task(_scheduler); - _executor->sync(); task.await(); } @@ -424,7 +420,6 @@ RPCNetwork::shutdown() _scheduler.Kill(_targetPoolTask.get()); _transport->ShutDown(true); _threadPool->Close(); - _executor->shutdown().sync(); } void diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.h b/messagebus/src/vespa/messagebus/network/rpcnetwork.h index e706431f90d..b95c0c77b3c 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.h @@ -65,12 +65,9 @@ private: std::unique_ptr<RPCTargetPool> _targetPool; std::unique_ptr<FNET_Task> _targetPoolTask; std::unique_ptr<RPCServicePool> _servicePool; - std::unique_ptr<vespalib::SyncableThreadExecutor> _executor; std::unique_ptr<RPCSendAdapter> _sendV2; SendAdapterMap _sendAdapters; CompressionConfig _compressionConfig; - bool _allowDispatchForEncode; - bool _allowDispatchForDecode; /** * Resolves and assigns a service address for the given recipient using the @@ -222,10 +219,6 @@ public: const slobrok::api::IMirrorAPI &getMirror() const override; CompressionConfig getCompressionConfig() { return _compressionConfig; } void invoke(FRT_RPCRequest *req); - vespalib::Executor & getExecutor() const { return *_executor; } - bool allowDispatchForEncode() const { return _allowDispatchForEncode; } - bool allowDispatchForDecode() const { return _allowDispatchForDecode; } - }; } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp index 8fcadaa64c6..fc060b48fc2 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp @@ -15,13 +15,10 @@ RPCNetworkParams::RPCNetworkParams(config::ConfigUri configUri) : _listenPort(0), _maxInputBufferSize(256_Ki), _maxOutputBufferSize(256_Ki), - _numThreads(4), _numNetworkThreads(1), _numRpcTargets(1), _events_before_wakeup(1), _tcpNoDelay(true), - _dispatchOnEncode(true), - _dispatchOnDecode(false), _connectionExpireSecs(600), _compressionConfig(CompressionConfig::LZ4, 6, 90, 1024) { } diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h index a8d611df653..4a4d92ba797 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h @@ -19,13 +19,10 @@ private: int _listenPort; uint32_t _maxInputBufferSize; uint32_t _maxOutputBufferSize; - uint32_t _numThreads; uint32_t _numNetworkThreads; uint32_t _numRpcTargets; uint32_t _events_before_wakeup; bool _tcpNoDelay; - bool _dispatchOnEncode; - bool _dispatchOnDecode; double _connectionExpireSecs; CompressionConfig _compressionConfig; @@ -113,19 +110,6 @@ public: return *this; } - /** - * Sets number of threads for the thread pool. - * - * @param numThreads number of threads for thread pool - * @return This, to allow chaining. - */ - RPCNetworkParams &setNumThreads(uint32_t numThreads) { - _numThreads = numThreads; - return *this; - } - - uint32_t getNumThreads() const { return _numThreads; } - RPCNetworkParams &setTcpNoDelay(bool tcpNoDelay) { _tcpNoDelay = tcpNoDelay; return *this; @@ -177,21 +161,6 @@ public: } CompressionConfig getCompressionConfig() const { return _compressionConfig; } - - RPCNetworkParams &setDispatchOnDecode(bool dispatchOnDecode) { - _dispatchOnDecode = dispatchOnDecode; - return *this; - } - - bool getDispatchOnDecode() const { return _dispatchOnDecode; } - - RPCNetworkParams &setDispatchOnEncode(bool dispatchOnEncode) { - _dispatchOnEncode = dispatchOnEncode; - return *this; - } - - bool getDispatchOnEncode() const { return _dispatchOnEncode; } - RPCNetworkParams &events_before_wakeup(uint32_t value) { _events_before_wakeup = value; return *this; diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.cpp b/messagebus/src/vespa/messagebus/network/rpcsend.cpp index 7627aa876b3..ff77a1bb639 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsend.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsend.cpp @@ -220,14 +220,7 @@ RPCSend::decode(vespalib::stringref protocolName, const vespalib::Version & vers void RPCSend::handleReply(Reply::UP reply) { - if (!_net->allowDispatchForEncode()) { - doHandleReply(std::move(reply)); - } else { - auto rejected = _net->getExecutor().execute(makeLambdaTask([this, reply = std::move(reply)]() mutable { - doHandleReply(std::move(reply)); - })); - assert (!rejected); - } + doHandleReply(std::move(reply)); } void @@ -256,15 +249,7 @@ void RPCSend::invoke(FRT_RPCRequest *req) { req->Detach(); - - if (!_net->allowDispatchForDecode()) { - doRequest(req); - } else { - auto rejected = _net->getExecutor().execute(makeLambdaTask([this, req]() { - doRequest(req); - })); - assert (!rejected); - } + doRequest(req); } void diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java index a36215e005f..ab2f53db863 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java @@ -2,7 +2,6 @@ package ai.vespa.models.evaluation; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.FileSource; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; @@ -10,7 +9,6 @@ import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; - import java.util.Map; import static org.junit.Assert.assertEquals; @@ -31,8 +29,8 @@ public class ModelTester { public Map<String, Model> models() { return models; } + @SuppressWarnings("deprecation") private static Map<String, Model> createModels(String path) { - RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId(path, "rank-profiles.cfg")); RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId(path, "ranking-constants.cfg")); RankingExpressionsConfig expressionsConfig = ConfigGetter.getConfig(RankingExpressionsConfig.class, fileConfigId(path, "ranking-expressions.cfg")); diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java index 540c534925e..c4e859bec9f 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java @@ -2,7 +2,6 @@ package ai.vespa.models.evaluation; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.FileSource; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; @@ -15,7 +14,6 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.yolean.Exceptions; import org.junit.Test; - import java.util.ArrayList; import java.util.List; @@ -128,6 +126,7 @@ public class ModelsEvaluatorTest { // TODO: Test argument-less function // TODO: Test with nested functions + @SuppressWarnings("deprecation") private ModelsEvaluator createModels() { RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java index 27d1c08ea39..992dae22aaf 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java @@ -51,6 +51,7 @@ public class OnnxEvaluatorTest { assertEquals(function.evaluate(), Tensor.from("tensor<float>(d0[2],d1[1]):[0.63931,0.67574]")); } + @SuppressWarnings("deprecation") private ModelsEvaluator createModels() { RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 7790f8a60d0..0de8ce5f061 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -257,6 +257,7 @@ public class ModelsEvaluationHandlerTest { handler.assertResponse(url, properties, 200, expected); } + @SuppressWarnings("deprecation") static private ModelsEvaluator createModels() { RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java index f065435ec15..8ab282668da 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java @@ -117,6 +117,7 @@ public class OnnxEvaluationHandlerTest { handler.assertResponse(url, properties, 200, expected); } + @SuppressWarnings("deprecation") static private ModelsEvaluator createModels() { RankProfilesConfig config = ConfigGetter.getConfig(RankProfilesConfig.class, fileConfigId("rank-profiles.cfg")); RankingConstantsConfig constantsConfig = ConfigGetter.getConfig(RankingConstantsConfig.class, fileConfigId("ranking-constants.cfg")); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java index 93195a3c340..b526c573c05 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java @@ -194,13 +194,28 @@ public class VespaServiceDumperImpl implements VespaServiceDumper { @Override public int servicePid() { if (pid == -1) { - ContainerPath findPidBinary = nodeAgentCtx.paths().underVespaHome("libexec/vespa/find-pid"); - CommandResult findPidResult = executeCommandInNode(List.of(findPidBinary.pathInContainer(), serviceId()), true); - this.pid = Integer.parseInt(findPidResult.getOutput()); + try { + pid = findServicePid(serviceId()); + } catch (RuntimeException e1) { + try { + // Workaround for Vespa 7 container clusters having service name 'qrserver' + if (serviceId().equals("container")) pid = findServicePid("qrserver"); + else throw e1; + } catch (RuntimeException e2) { + e1.addSuppressed(e2); + throw e1; + } + } } return pid; } + private int findServicePid(String serviceId) { + ContainerPath findPidBinary = nodeAgentCtx.paths().underVespaHome("libexec/vespa/find-pid"); + CommandResult findPidResult = executeCommandInNode(List.of(findPidBinary.pathInContainer(), serviceId), true); + return Integer.parseInt(findPidResult.getOutput()); + } + @Override public CommandResult executeCommandInNode(List<String> command, boolean logOutput) { CommandResult result = container.executeCommandInContainer(nodeAgentCtx, nodeAgentCtx.users().vespa(), command.toArray(new String[0])); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 9d9a1304418..368a8da0f90 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; import com.yahoo.lang.MutableInteger; -import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.JacksonFlag; import com.yahoo.vespa.flags.ListFlag; @@ -77,16 +76,14 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { @Override protected double maintain() { - try (Mutex lock = nodeRepository().nodes().lockUnallocated()) { - NodeList nodes = nodeRepository().nodes().list(); - resumeProvisioning(nodes, lock); - convergeToCapacity(nodes); - } + NodeList nodes = nodeRepository().nodes().list(); + resumeProvisioning(nodes); + convergeToCapacity(nodes); return 1.0; } /** Resume provisioning of already provisioned hosts and their children */ - private void resumeProvisioning(NodeList nodes, Mutex lock) { + private void resumeProvisioning(NodeList nodes) { Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant, NodeType.config, NodeType.controller) .asList() @@ -97,9 +94,11 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { nodes.state(Node.State.provisioned).nodeType(NodeType.host, NodeType.confighost, NodeType.controllerhost).forEach(host -> { Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of()); try { - List<Node> updatedNodes = hostProvisioner.provision(host, children); - verifyDns(updatedNodes); - nodeRepository().nodes().write(updatedNodes, lock); + try (var lock = nodeRepository().nodes().lockUnallocated()) { + List<Node> updatedNodes = hostProvisioner.provision(host, children); + verifyDns(updatedNodes); + nodeRepository().nodes().write(updatedNodes, lock); + } } catch (IllegalArgumentException | IllegalStateException e) { log.log(Level.INFO, "Could not provision " + host.hostname() + " with " + children.size() + " children, will retry in " + interval() + ": " + Exceptions.toMessageString(e)); @@ -189,17 +188,12 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { private List<Node> candidatesForRemoval(List<Node> nodes) { Map<String, Node> hostsByHostname = new HashMap<>(nodes.stream() - .filter(node -> { - switch (node.type()) { - case host: - // TODO: Mark empty tenant hosts as wanttoretire & wanttodeprovision elsewhere, then handle as confighost here - return node.state() != Node.State.parked || node.status().wantToDeprovision(); - case confighost: - case controllerhost: - return node.state() == Node.State.parked && node.status().wantToDeprovision(); - default: - return false; - } + .filter(node -> switch (node.type()) { + case host -> + // TODO: Mark empty tenant hosts as wanttoretire & wanttodeprovision elsewhere, then handle as confighost here + node.state() != Node.State.parked || node.status().wantToDeprovision(); + case confighost, controllerhost -> node.state() == Node.State.parked && node.status().wantToDeprovision(); + default -> false; }) .collect(Collectors.toMap(Node::hostname, Function.identity()))); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java index ac804f99cd3..c2d4506a28c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.node; import com.google.common.collect.ImmutableMap; import com.yahoo.vespa.hosted.provision.Node; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -50,6 +51,12 @@ public class History { return builder.build(); } + /** Returns the age of this node as best as we can determine: The time since the first event registered for it */ + public Duration age(Instant now) { + Instant oldestEventTime = events.values().stream().map(event -> event.at()).sorted().findFirst().orElse(now); + return Duration.between(oldestEventTime, now); + } + /** Returns the last event of given type, if it is present in this history */ public Optional<Event> event(Event.Type type) { return Optional.ofNullable(events.get(type)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java index 8578e3eb5ec..2b790ff7392 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java @@ -30,4 +30,5 @@ public class NodeListFilter { public static Predicate<Node> from(List<Node> nodes) { return makePredicate(nodes); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java index 30fd2713017..4178d4a6328 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter; +import java.time.Instant; import java.util.Objects; import java.util.Optional; import java.util.logging.Logger; @@ -39,8 +40,10 @@ public class DelegatingOsUpgrader implements OsUpgrader { public void upgradeTo(OsVersionTarget target) { NodeList activeNodes = nodeRepository.nodes().list(Node.State.active).nodeType(target.nodeType()); int numberToUpgrade = Math.max(0, maxActiveUpgrades - activeNodes.changingOsVersionTo(target.version()).size()); + Instant now = nodeRepository.clock().instant(); NodeList nodesToUpgrade = activeNodes.not().changingOsVersionTo(target.version()) .osVersionIsBefore(target.version()) + .matching(node -> canUpgradeAt(now, node)) .byIncreasingOsVersion() .first(numberToUpgrade); if (nodesToUpgrade.size() == 0) return; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java index 5310ef339ed..4140de76368 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java @@ -2,6 +2,9 @@ package com.yahoo.vespa.hosted.provision.os; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; + +import java.time.Instant; /** * Interface for an OS upgrader. @@ -16,4 +19,9 @@ public interface OsUpgrader { /** Disable OS upgrade for all nodes of given type */ void disableUpgrade(NodeType type); + /** Returns whether node can upgrade at given instant */ + default boolean canUpgradeAt(Instant instant, Node node) { + return true; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java index 7c6d1cb69db..440046ab818 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java @@ -84,7 +84,7 @@ public class OsVersions { Version target = Optional.ofNullable(change.targets().get(nodeType)) .map(OsVersionTarget::version) .orElse(Version.emptyVersion); - chooseUpgrader(nodeType, target).disableUpgrade(nodeType); + chooseUpgrader(nodeType, Optional.of(target)).disableUpgrade(nodeType); return change.withoutTarget(nodeType); }); } @@ -120,7 +120,7 @@ public class OsVersions { try (Lock lock = db.lockOsVersionChange()) { OsVersionTarget target = readChange().targets().get(nodeType); if (target == null) return; // No target set for this type - OsUpgrader upgrader = chooseUpgrader(nodeType, target.version()); + OsUpgrader upgrader = chooseUpgrader(nodeType, Optional.of(target.version())); if (resume) { upgrader.upgradeTo(target); } else { @@ -129,17 +129,23 @@ public class OsVersions { } } + /** Returns whether node can be upgraded now */ + public boolean canUpgrade(Node node) { + return chooseUpgrader(node.type(), Optional.empty()).canUpgradeAt(nodeRepository.clock().instant(), node); + } + /** Returns the upgrader to use when upgrading given node type to target */ - private OsUpgrader chooseUpgrader(NodeType nodeType, Version target) { + private OsUpgrader chooseUpgrader(NodeType nodeType, Optional<Version> target) { if (reprovisionToUpgradeOs) { return new RetiringOsUpgrader(nodeRepository); } // Require rebuild if we have any nodes of this type on a major version lower than target - boolean rebuildRequired = nodeRepository.nodes().list(Node.State.active).nodeType(nodeType).stream() + boolean rebuildRequired = target.isPresent() && + nodeRepository.nodes().list(Node.State.active).nodeType(nodeType).stream() .map(Node::status) .map(Status::osVersion) .anyMatch(osVersion -> osVersion.current().isPresent() && - osVersion.current().get().getMajor() < target.getMajor()); + osVersion.current().get().getMajor() < target.get().getMajor()); if (rebuildRequired) { return new RebuildingOsUpgrader(nodeRepository); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java index efc377e6cc3..f96effe9e10 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java @@ -47,7 +47,7 @@ public class RebuildingOsUpgrader implements OsUpgrader { public void upgradeTo(OsVersionTarget target) { NodeList allNodes = nodeRepository.nodes().list(); Instant now = nodeRepository.clock().instant(); - rebuildableHosts(target, allNodes).forEach(host -> rebuild(host, target.version(), now)); + rebuildableHosts(target, allNodes, now).forEach(host -> rebuild(host, target.version(), now)); } @Override @@ -62,7 +62,7 @@ public class RebuildingOsUpgrader implements OsUpgrader { return Math.max(0, limit - hostsOfType.rebuilding().size()); } - private List<Node> rebuildableHosts(OsVersionTarget target, NodeList allNodes) { + private List<Node> rebuildableHosts(OsVersionTarget target, NodeList allNodes, Instant now) { NodeList hostsOfTargetType = allNodes.nodeType(target.nodeType()); int rebuildLimit = rebuildLimit(target.nodeType(), hostsOfTargetType); @@ -76,6 +76,7 @@ public class RebuildingOsUpgrader implements OsUpgrader { NodeList candidates = hostsOfTargetType.state(Node.State.active) .not().rebuilding() .osVersionIsBefore(target.version()) + .matching(node -> canUpgradeAt(now, node)) .byIncreasingOsVersion(); for (Node host : candidates) { if (hostsToRebuild.size() == rebuildLimit) break; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java index d923c78a929..79b7441cc34 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java @@ -26,6 +26,9 @@ public class RetiringOsUpgrader implements OsUpgrader { private static final Logger LOG = Logger.getLogger(RetiringOsUpgrader.class.getName()); + /** The duration this leaves new nodes alone before scheduling any upgrade */ + static final Duration GRACE_PERIOD = Duration.ofDays(30); + protected final NodeRepository nodeRepository; public RetiringOsUpgrader(NodeRepository nodeRepository) { @@ -33,21 +36,27 @@ public class RetiringOsUpgrader implements OsUpgrader { } @Override - public final void upgradeTo(OsVersionTarget target) { + public void upgradeTo(OsVersionTarget target) { NodeList allNodes = nodeRepository.nodes().list(); Instant now = nodeRepository.clock().instant(); NodeList candidates = candidates(now, target, allNodes); candidates.not().deprovisioning() + .matching(node -> canUpgradeAt(now, node)) .byIncreasingOsVersion() .first(1) .forEach(node -> deprovision(node, target.version(), now)); } @Override - public final void disableUpgrade(NodeType type) { + public void disableUpgrade(NodeType type) { // No action needed in this implementation. } + @Override + public boolean canUpgradeAt(Instant instant, Node node) { + return node.history().age(instant).compareTo(GRACE_PERIOD) > 0; + } + /** Returns nodes that are candidates for upgrade */ private NodeList candidates(Instant instant, OsVersionTarget target, NodeList allNodes) { NodeList activeNodes = allNodes.state(Node.State.active).nodeType(target.nodeType()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 3659166c9da..efd76187bc6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -161,7 +161,10 @@ class NodesResponse extends SlimeJsonResponse { object.setLong("rebootGeneration", node.status().reboot().wanted()); object.setLong("currentRebootGeneration", node.status().reboot().current()); node.status().osVersion().current().ifPresent(version -> object.setString("currentOsVersion", version.toFullString())); - node.status().osVersion().wanted().ifPresent(version -> object.setString("wantedOsVersion", version.toFullString())); + node.status().osVersion().wanted().ifPresent(version -> { + object.setString("wantedOsVersion", version.toFullString()); + object.setBool("deferOsUpgrade", !nodeRepository.osVersions().canUpgrade(node)); + }); node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong("currentFirmwareCheck", instant.toEpochMilli())); if (node.type().isHost()) nodeRepository.firmwareChecks().requiredAfter().ifPresent(after -> object.setLong("wantedFirmwareCheck", after.toEpochMilli())); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 97a8ac0d655..af2a215dae0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -172,6 +172,7 @@ public class OsVersionsTest { Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list() .hosts() .not().state(Node.State.deprovisioned); + tester.clock().advance(RetiringOsUpgrader.GRACE_PERIOD.plusDays(1)); // Target is set and upgrade started var version1 = Version.fromString("7.1"); @@ -233,6 +234,7 @@ public class OsVersionsTest { Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list() .nodeType(NodeType.confighost) .not().state(Node.State.deprovisioned); + tester.clock().advance(RetiringOsUpgrader.GRACE_PERIOD.plusDays(1)); // Target is set with zero budget and upgrade started var version1 = Version.fromString("7.1"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 6b1853b3893..19af4d00e54 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -770,8 +770,9 @@ public class NodesV2ApiTest { Request.Method.PATCH), "{\"message\":\"Set osVersion to 7.5.2, upgradeBudget to PT0S for nodes of type host\"}"); + var nodeRepository = (NodeRepository) tester.container().components().getComponent(MockNodeRepository.class.getName()); + // Activate target - var nodeRepository = (NodeRepository)tester.container().components().getComponent(MockNodeRepository.class.getName()); var osUpgradeActivator = new OsUpgradeActivator(nodeRepository, Duration.ofDays(1), new TestMetric()); osUpgradeActivator.run(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json index b885f7bd7fc..287db73faf6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json @@ -30,6 +30,7 @@ "currentRebootGeneration": 0, "currentOsVersion": "7.5.2", "wantedOsVersion": "7.5.2", + "deferOsUpgrade": false, "failCount": 0, "wantToRetire": false, "preferToRetire": false, diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java index aa218755792..f798cea3572 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java @@ -47,7 +47,7 @@ public class OrchestratorContext implements AutoCloseable { /** Create an OrchestratorContext for an operation on a single application. */ public static OrchestratorContext createContextForSingleAppOp(Clock clock) { return new OrchestratorContext(null, clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_SINGLE_OP), - false, false); + false, false); } public static OrchestratorContext createContextForAdminOp(Clock clock) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index b3244c1ac74..587875363d5 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator; -import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; @@ -19,6 +19,7 @@ import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState; import com.yahoo.vespa.orchestrator.model.ApplicationApi; import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory; +import com.yahoo.vespa.orchestrator.model.ContentService; import com.yahoo.vespa.orchestrator.model.NodeGroup; import com.yahoo.vespa.orchestrator.model.VespaModelUtil; import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; @@ -99,7 +100,8 @@ public class OrchestratorImpl implements Orchestrator { { this(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource, zone), clusterControllerClientFactory, - applicationApiFactory), + applicationApiFactory, + flagSource), clusterControllerClientFactory, statusService, serviceMonitor, @@ -425,7 +427,7 @@ public class OrchestratorImpl implements Orchestrator { ClusterControllerClient client = clusterControllerClientFactory.createClient(clusterControllers, cluster.clusterId().s()); for (ServiceInstance service : cluster.serviceInstances()) { try { - if ( ! client.trySetNodeState(context, service.hostName(), VespaModelUtil.getStorageNodeIndex(service.configId()), MAINTENANCE)) + if ( ! client.trySetNodeState(context, service.hostName(), VespaModelUtil.getStorageNodeIndex(service.configId()), MAINTENANCE, ContentService.STORAGE_NODE, false)) return false; } catch (Exception e) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java index 98ca9f805b4..e563f36a488 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java @@ -5,6 +5,7 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.ApplicationStateChangeDeniedException; import com.yahoo.vespa.orchestrator.OrchestratorContext; +import com.yahoo.vespa.orchestrator.model.ContentService; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; /** @@ -19,7 +20,8 @@ public interface ClusterControllerClient { * @throws HostStateChangeDeniedException if operation fails, or is otherwise disallowed. */ boolean trySetNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, - ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException; + ClusterControllerNodeState wantedState, ContentService contentService, boolean force) + throws HostStateChangeDeniedException; /** * Requests that a cluster controller sets the requested node to the requested state. @@ -27,7 +29,8 @@ public interface ClusterControllerClient { * @throws HostStateChangeDeniedException if operation fails, or is disallowed. */ void setNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, - ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException; + ClusterControllerNodeState wantedState, ContentService contentService, boolean force) + throws HostStateChangeDeniedException; /** * Requests that a cluster controller sets all nodes in the cluster to the requested state. diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java index 28ba259d2ae..25567cb00a1 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.ApplicationStateChangeDeniedException; import com.yahoo.vespa.orchestrator.OrchestratorContext; +import com.yahoo.vespa.orchestrator.model.ContentService; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; import com.yahoo.yolean.Exceptions; @@ -53,14 +54,16 @@ public class ClusterControllerClientImpl implements ClusterControllerClient { } private boolean setNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, - ClusterControllerNodeState wantedState, boolean throwOnFailure) { + ClusterControllerNodeState wantedState, ContentService contentService, + Condition condition, boolean throwOnFailure) { try { ClusterControllerClientTimeouts timeouts = context.getClusterControllerTimeouts(); Inspector response = client.send(strategy(hosts), Method.POST) - .at("cluster", "v2", clusterName, "storage", Integer.toString(storageNodeIndex)) + .at("cluster", "v2", clusterName, contentService.nameInClusterController(), + Integer.toString(storageNodeIndex)) .deadline(timeouts.readBudget()) .parameters(() -> deadline(timeouts)) - .body(stateChangeRequestBytes(wantedState, Condition.SAFE, context.isProbe())) + .body(stateChangeRequestBytes(wantedState, condition, context.isProbe())) .throwing(retryOnRedirect) .read(SlimeUtils::jsonToSlime).get(); if ( ! response.field("wasModified").asBool()) { @@ -99,13 +102,17 @@ public class ClusterControllerClientImpl implements ClusterControllerClient { } @Override - public boolean trySetNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException { - return setNodeState(context, host, storageNodeIndex, wantedState, false); + public boolean trySetNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, + ClusterControllerNodeState wantedState, ContentService contentService, boolean force) + throws HostStateChangeDeniedException { + return setNodeState(context, host, storageNodeIndex, wantedState, contentService, force ? Condition.FORCE : Condition.SAFE, false); } @Override - public void setNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException { - setNodeState(context, host, storageNodeIndex, wantedState, true); + public void setNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, + ClusterControllerNodeState wantedState, ContentService contentService, boolean force) + throws HostStateChangeDeniedException { + setNodeState(context, host, storageNodeIndex, wantedState, contentService, force ? Condition.FORCE : Condition.SAFE, true); } @Override diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ContentService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ContentService.java new file mode 100644 index 00000000000..f611bada264 --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ContentService.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.model; + +public enum ContentService { + DISTRIBUTOR("distributor"), STORAGE_NODE("storage"); + + private final String nameInClusterController; + + ContentService(String nameInClusterController) { + this.nameInClusterController = nameInClusterController; + } + + public String nameInClusterController() { return nameInClusterController; } +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNode.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNode.java index 863c817fe6f..671bd351b3b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNode.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNode.java @@ -8,5 +8,6 @@ import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; public interface StorageNode extends Comparable<StorageNode> { HostName hostName(); - void setNodeState(OrchestratorContext context, ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException; + void setStorageNodeState(OrchestratorContext context, ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException; + void forceDistributorState(OrchestratorContext context, ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java index f3e3fd0e674..29f7700ef54 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java @@ -41,7 +41,17 @@ public class StorageNodeImpl implements StorageNode { } @Override - public void setNodeState(OrchestratorContext context, ClusterControllerNodeState wantedNodeState) + public void setStorageNodeState(OrchestratorContext context, ClusterControllerNodeState wantedNodeState) + throws HostStateChangeDeniedException { + setNodeState(context, wantedNodeState, ContentService.STORAGE_NODE, false); + } + + @Override + public void forceDistributorState(OrchestratorContext context, ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException { + setNodeState(context, wantedState, ContentService.DISTRIBUTOR, true); + } + + public void setNodeState(OrchestratorContext context, ClusterControllerNodeState wantedNodeState, ContentService contentService, boolean force) throws HostStateChangeDeniedException { // The "cluster name" used by the Cluster Controller IS the cluster ID. String clusterId = this.clusterId.s(); @@ -52,17 +62,18 @@ public class StorageNodeImpl implements StorageNode { clusterControllers, clusterId); - ConfigId configId = storageService.configId(); - int nodeIndex = VespaModelUtil.getStorageNodeIndex(configId); + int nodeIndex = VespaModelUtil.getStorageNodeIndex(storageService.configId()); - logger.log(Level.FINE, () -> "Setting cluster controller state for " + - "application " + applicationInstance.reference().asString() + - ", host " + hostName() + - ", cluster name " + clusterId + - ", node index " + nodeIndex + - ", node state " + wantedNodeState); + logger.log(Level.FINE, () -> (force ? "Force" : "Safe") + + " setting cluster controller state for " + + "application " + applicationInstance.reference().asString() + + ", host " + hostName() + + ", cluster name " + clusterId + + ", service " + contentService.nameInClusterController() + + ", node index " + nodeIndex + + ", node state " + wantedNodeState); - client.setNodeState(context, storageService.hostName(), nodeIndex, wantedNodeState); + client.setNodeState(context, storageService.hostName(), nodeIndex, wantedNodeState, contentService, force); } @Override diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java index 2ce62081f51..3f23be5e514 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java @@ -3,6 +3,10 @@ package com.yahoo.vespa.orchestrator.policy; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState; @@ -30,13 +34,16 @@ public class HostedVespaPolicy implements Policy { private final HostedVespaClusterPolicy clusterPolicy; private final ClusterControllerClientFactory clusterControllerClientFactory; private final ApplicationApiFactory applicationApiFactory; + private final BooleanFlag keepStorageNodeUpFlag; public HostedVespaPolicy(HostedVespaClusterPolicy clusterPolicy, ClusterControllerClientFactory clusterControllerClientFactory, - ApplicationApiFactory applicationApiFactory) { + ApplicationApiFactory applicationApiFactory, + FlagSource flagSource) { this.clusterPolicy = clusterPolicy; this.clusterControllerClientFactory = clusterControllerClientFactory; this.applicationApiFactory = applicationApiFactory; + this.keepStorageNodeUpFlag = Flags.KEEP_STORAGE_NODE_UP.bindTo(flagSource); } @Override @@ -52,7 +59,7 @@ public class HostedVespaPolicy implements Policy { // Ask Cluster Controller to set storage nodes in maintenance, unless the node is already allowed // to be down (or permanently down) in case they are guaranteed to be in maintenance already. for (StorageNode storageNode : application.getNoRemarksStorageNodesInGroupInClusterOrder()) { - storageNode.setNodeState(context, ClusterControllerNodeState.MAINTENANCE); + storageNode.setStorageNodeState(context, ClusterControllerNodeState.MAINTENANCE); } // Ensure all nodes in the group are marked as allowed to be down @@ -68,7 +75,7 @@ public class HostedVespaPolicy implements Policy { throws HostStateChangeDeniedException { // Always defer to Cluster Controller whether it's OK to resume storage node for (StorageNode storageNode : application.getSuspendedStorageNodesInGroupInReverseClusterOrder()) { - storageNode.setNodeState(context, ClusterControllerNodeState.UP); + storageNode.setStorageNodeState(context, ClusterControllerNodeState.UP); } // In particular, we're not modifying the state of PERMANENTLY_DOWN nodes. @@ -94,10 +101,18 @@ public class HostedVespaPolicy implements Policy { clusterPolicy.verifyGroupGoingDownPermanentlyIsFine(cluster); } - // Ask Cluster Controller to set storage nodes to DOWN. - // These storage nodes are guaranteed to be NO_REMARKS + boolean keepStorageNodeUp = keepStorageNodeUpFlag + .with(FetchVector.Dimension.APPLICATION_ID, applicationApi.applicationId().serializedForm()) + .value(); + + // Get permission from the Cluster Controller to remove the content nodes. for (StorageNode storageNode : applicationApi.getStorageNodesInGroupInClusterOrder()) { - storageNode.setNodeState(context, ClusterControllerNodeState.DOWN); + if (keepStorageNodeUp) { + storageNode.setStorageNodeState(context.createSubcontextForSingleAppOp(true), ClusterControllerNodeState.DOWN); + storageNode.forceDistributorState(context, ClusterControllerNodeState.DOWN); + } else { + storageNode.setStorageNodeState(context, ClusterControllerNodeState.DOWN); + } } // Ensure all nodes in the group are marked as permanently down diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index f5b7771d5f4..9fd7b9dade9 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState; import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory; +import com.yahoo.vespa.orchestrator.model.ContentService; import com.yahoo.vespa.orchestrator.model.NodeGroup; import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; @@ -104,7 +105,8 @@ public class OrchestratorImplTest { clustercontroller = new ClusterControllerClientFactoryMock(); orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource, zone), clustercontroller, - applicationApiFactory), + applicationApiFactory, + flagSource), clustercontroller, statusService, new DummyServiceMonitor(), @@ -450,7 +452,8 @@ public class OrchestratorImplTest { orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource, zone), clusterControllerClientFactory, - applicationApiFactory), + applicationApiFactory, + flagSource), clusterControllerClientFactory, statusService, serviceMonitor, @@ -459,16 +462,16 @@ public class OrchestratorImplTest { applicationApiFactory, flagSource); - when(fooClient.trySetNodeState(any(), any(), eq(1), eq(ClusterControllerNodeState.MAINTENANCE))).thenReturn(true); - when(fooClient.trySetNodeState(any(), any(), eq(2), eq(ClusterControllerNodeState.MAINTENANCE))).thenReturn(true); - when(barClient.trySetNodeState(any(), any(), eq(0), eq(ClusterControllerNodeState.MAINTENANCE))).thenReturn(true); - when(barClient.trySetNodeState(any(), any(), eq(3), eq(ClusterControllerNodeState.MAINTENANCE))).thenReturn(true); + when(fooClient.trySetNodeState(any(), any(), eq(1), eq(ClusterControllerNodeState.MAINTENANCE), eq(ContentService.STORAGE_NODE), eq(false))).thenReturn(true); + when(fooClient.trySetNodeState(any(), any(), eq(2), eq(ClusterControllerNodeState.MAINTENANCE), eq(ContentService.STORAGE_NODE), eq(false))).thenReturn(true); + when(barClient.trySetNodeState(any(), any(), eq(0), eq(ClusterControllerNodeState.MAINTENANCE), eq(ContentService.STORAGE_NODE), eq(false))).thenReturn(true); + when(barClient.trySetNodeState(any(), any(), eq(3), eq(ClusterControllerNodeState.MAINTENANCE), eq(ContentService.STORAGE_NODE), eq(false))).thenReturn(true); assertTrue(orchestrator.isQuiescent(id)); - when(fooClient.trySetNodeState(any(), any(), eq(2), eq(ClusterControllerNodeState.MAINTENANCE))).thenReturn(false); + when(fooClient.trySetNodeState(any(), any(), eq(2), eq(ClusterControllerNodeState.MAINTENANCE), eq(ContentService.STORAGE_NODE), eq(false))).thenReturn(false); assertFalse(orchestrator.isQuiescent(id)); - when(fooClient.trySetNodeState(any(), any(), eq(2), eq(ClusterControllerNodeState.MAINTENANCE))).thenThrow(new RuntimeException()); + when(fooClient.trySetNodeState(any(), any(), eq(2), eq(ClusterControllerNodeState.MAINTENANCE), eq(ContentService.STORAGE_NODE), eq(false))).thenThrow(new RuntimeException()); assertFalse(orchestrator.isQuiescent(id)); } @@ -509,7 +512,8 @@ public class OrchestratorImplTest { orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource, zone), clusterControllerClientFactory, - applicationApiFactory), + applicationApiFactory, + flagSource), clusterControllerClientFactory, statusService, serviceMonitor, diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java index 29d6463a7d7..323ae678b0b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java @@ -57,7 +57,8 @@ public class OrchestratorTest { var timer = new TestTimer(); var clustercontroller = new ClusterControllerClientFactoryMock(); var applicationApiFactory = new ApplicationApiFactory(3, 5, timer.toUtcClock()); - var policy = new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource, zone), clustercontroller, applicationApiFactory); + var clusterPolicy = new HostedVespaClusterPolicy(flagSource, zone); + var policy = new HostedVespaPolicy(clusterPolicy, clustercontroller, applicationApiFactory, flagSource); var zone = new Zone(SystemName.cd, Environment.prod, RegionName.from("cd-us-east-1")); this.superModelManager = new MySuperModelProvider(); var duperModel = new DuperModel(); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java index 597487ccfc5..a5348ad3d07 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java @@ -5,11 +5,10 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.orchestrator.ApplicationStateChangeDeniedException; import com.yahoo.vespa.orchestrator.DummyServiceMonitor; import com.yahoo.vespa.orchestrator.OrchestratorContext; +import com.yahoo.vespa.orchestrator.model.ContentService; import com.yahoo.vespa.orchestrator.model.VespaModelUtil; -import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import java.util.HashMap; import java.util.List; @@ -30,7 +29,7 @@ public class ClusterControllerClientFactoryMock implements ClusterControllerClie try { ClusterId clusterName = VespaModelUtil.getContentClusterName(appInstance, hostName); int storageNodeIndex = VespaModelUtil.getStorageNodeIndex(appInstance, hostName); - String globalMapKey = clusterName + "/" + storageNodeIndex; + String globalMapKey = clusterName + "/" + ContentService.STORAGE_NODE.nameInClusterController() + "/" + storageNodeIndex; return nodes.getOrDefault(globalMapKey, ClusterControllerNodeState.UP) == ClusterControllerNodeState.MAINTENANCE; } catch (Exception e) { //Catch all - meant to catch cases where the node is not part of a storage cluster @@ -44,7 +43,7 @@ public class ClusterControllerClientFactoryMock implements ClusterControllerClie for (HostName host : hosts) { ClusterId clusterName = VespaModelUtil.getContentClusterName(app, host); int storageNodeIndex = VespaModelUtil.getStorageNodeIndex(app, host); - String globalMapKey = clusterName + "/" + storageNodeIndex; + String globalMapKey = clusterName + "/" + ContentService.STORAGE_NODE.nameInClusterController() + "/" + storageNodeIndex; nodes.put(globalMapKey, ClusterControllerNodeState.UP); } } @@ -53,12 +52,12 @@ public class ClusterControllerClientFactoryMock implements ClusterControllerClie @Override public ClusterControllerClient createClient(List<HostName> clusterControllers, String clusterName) { return new ClusterControllerClient() { - @Override public boolean trySetNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, ClusterControllerNodeState wantedState) { - nodes.put(clusterName + "/" + storageNodeIndex, wantedState); + @Override public boolean trySetNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, ClusterControllerNodeState wantedState, ContentService contentService, boolean force) { + nodes.put(clusterName + "/" + contentService.nameInClusterController() + "/" + storageNodeIndex, wantedState); return true; } - @Override public void setNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, ClusterControllerNodeState wantedState) { - trySetNodeState(context, host, storageNodeIndex, wantedState); + @Override public void setNodeState(OrchestratorContext context, HostName host, int storageNodeIndex, ClusterControllerNodeState wantedState, ContentService contentService, boolean force) { + trySetNodeState(context, host, storageNodeIndex, wantedState, contentService, false); } @Override public void setApplicationState(OrchestratorContext context, ApplicationInstanceId applicationId, ClusterControllerNodeState wantedState) { nodes.replaceAll((key, state) -> key.startsWith(clusterName + "/") ? wantedState : state); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImplTest.java index b8e078c60d4..81615f59be9 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImplTest.java @@ -7,13 +7,13 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.ApplicationStateChangeDeniedException; import com.yahoo.vespa.orchestrator.OrchestratorContext; +import com.yahoo.vespa.orchestrator.model.ContentService; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.time.Clock; import java.time.Duration; import java.util.List; @@ -66,7 +66,7 @@ public class ClusterControllerClientImplTest { return "{ \"wasModified\": true }"; }, 200); - client.setNodeState(context, host, 2, DOWN); + client.setNodeState(context, host, 2, DOWN, ContentService.STORAGE_NODE, false); clock.advance(Duration.ofSeconds(9)); wire.expect((url, body) -> { @@ -79,7 +79,7 @@ public class ClusterControllerClientImplTest { 200); assertEquals("Changing the state of node would violate controller-set-node-state: Failed to set state to DOWN in cluster controller: because", assertThrows(HostStateChangeDeniedException.class, - () -> client.setNodeState(context, host, 1, DOWN)) + () -> client.setNodeState(context, host, 1, DOWN, ContentService.STORAGE_NODE, false)) .getMessage()); } @@ -93,7 +93,7 @@ public class ClusterControllerClientImplTest { return "{ \"wasModified\": false, \"reason\": \"no reason\" }"; }, 200); - assertFalse(client.trySetNodeState(OrchestratorContext.createContextForBatchProbe(clock), host, 2, MAINTENANCE)); + assertFalse(client.trySetNodeState(OrchestratorContext.createContextForBatchProbe(clock), host, 2, MAINTENANCE, ContentService.STORAGE_NODE, false)); } @Test @@ -134,7 +134,7 @@ public class ClusterControllerClientImplTest { assertEquals("Changing the state of node would violate deadline: Timeout while waiting for setNodeState(2, UP) " + "against [host1, host2, host3]: Timed out after PT10S", assertThrows(HostStateChangeDeniedException.class, - () -> client.setNodeState(context, host, 2, UP)) + () -> client.setNodeState(context, host, 2, UP, ContentService.STORAGE_NODE, false)) .getMessage()); } @@ -154,7 +154,7 @@ public class ClusterControllerClientImplTest { assertEquals("Changing the state of node would violate controller-set-node-state: Failed setting node 2 in cluster cc to state UP: " + "got status code 503 for POST http://host1:19050/cluster/v2/cc/storage/2?timeout=9.6", assertThrows(HostStateChangeDeniedException.class, - () -> client.setNodeState(context, host, 2, UP)) + () -> client.setNodeState(context, host, 2, UP, ContentService.STORAGE_NODE, false)) .getMessage()); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java index 291f96e3dc3..f2e2972ae9f 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java @@ -80,9 +80,9 @@ class ModelTestUtils { mock(Metric.class), new TestTimer(), new DummyAntiServiceMonitor()); - private final Orchestrator orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(flagSource, Zone.defaultZone()), - clusterControllerClientFactory, - applicationApiFactory()), + private final HostedVespaClusterPolicy clusterPolicy = new HostedVespaClusterPolicy(flagSource, Zone.defaultZone()); + private final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clusterControllerClientFactory, applicationApiFactory(), flagSource); + private final Orchestrator orchestrator = new OrchestratorImpl(policy, clusterControllerClientFactory, statusService, serviceMonitor, diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java index f01ce5a2227..a622142b873 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java @@ -5,6 +5,7 @@ package com.yahoo.vespa.orchestrator.policy; import com.yahoo.config.provision.ApplicationId; import com.yahoo.test.ManualClock; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient; @@ -37,6 +38,7 @@ public class HostedVespaPolicyTest { private final ClusterControllerClient client = mock(ClusterControllerClient.class); private final ManualClock clock = new ManualClock(); private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, clock); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); @Before public void setUp() { @@ -47,7 +49,7 @@ public class HostedVespaPolicyTest { public void testGrantSuspension() throws HostStateChangeDeniedException { final HostedVespaClusterPolicy clusterPolicy = mock(HostedVespaClusterPolicy.class); when(clusterPolicy.verifyGroupGoingDownIsFine(any())).thenReturn(SuspensionReasons.nothingNoteworthy()); - final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory, applicationApiFactory); + final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory, applicationApiFactory, flagSource); final ApplicationApi applicationApi = mock(ApplicationApi.class); when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("tenant:app:default")); @@ -85,8 +87,8 @@ public class HostedVespaPolicyTest { order.verify(clusterPolicy).verifyGroupGoingDownIsFine(clusterApi3); order.verify(applicationApi).getNoRemarksStorageNodesInGroupInClusterOrder(); - order.verify(storageNode1).setNodeState(context, ClusterControllerNodeState.MAINTENANCE); - order.verify(storageNode3).setNodeState(context, ClusterControllerNodeState.MAINTENANCE); + order.verify(storageNode1).setStorageNodeState(context, ClusterControllerNodeState.MAINTENANCE); + order.verify(storageNode3).setStorageNodeState(context, ClusterControllerNodeState.MAINTENANCE); order.verify(applicationApi).getNodesInGroupWithStatus(HostStatus.NO_REMARKS); order.verify(applicationApi).setHostState(context, hostName1, HostStatus.ALLOWED_TO_BE_DOWN); @@ -99,7 +101,7 @@ public class HostedVespaPolicyTest { @Test public void testAcquirePermissionToRemove() throws OrchestrationException { final HostedVespaClusterPolicy clusterPolicy = mock(HostedVespaClusterPolicy.class); - final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory, applicationApiFactory); + final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory, applicationApiFactory, flagSource); final ApplicationApi applicationApi = mock(ApplicationApi.class); when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("tenant:app:default")); @@ -128,6 +130,8 @@ public class HostedVespaPolicyTest { InOrder order = inOrder(applicationApi, clusterPolicy, storageNode1, storageNode3); OrchestratorContext context = mock(OrchestratorContext.class); + OrchestratorContext probeContext = mock(OrchestratorContext.class); + when(context.createSubcontextForSingleAppOp(true)).thenReturn(probeContext); policy.acquirePermissionToRemove(context, applicationApi); order.verify(applicationApi).getClusters(); @@ -136,8 +140,8 @@ public class HostedVespaPolicyTest { order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi3); order.verify(applicationApi).getStorageNodesInGroupInClusterOrder(); - order.verify(storageNode1).setNodeState(context, ClusterControllerNodeState.DOWN); - order.verify(storageNode3).setNodeState(context, ClusterControllerNodeState.DOWN); + order.verify(storageNode1).setStorageNodeState(probeContext, ClusterControllerNodeState.DOWN); + order.verify(storageNode3).setStorageNodeState(probeContext, ClusterControllerNodeState.DOWN); order.verify(applicationApi).getNodesInGroupWith(any()); order.verify(applicationApi).setHostState(context, hostName1, HostStatus.PERMANENTLY_DOWN); @@ -150,7 +154,7 @@ public class HostedVespaPolicyTest { @Test public void testAcquirePermissionToRemoveConfigServer() throws OrchestrationException { final HostedVespaClusterPolicy clusterPolicy = mock(HostedVespaClusterPolicy.class); - final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory, applicationApiFactory); + final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory, applicationApiFactory, flagSource); final ApplicationApi applicationApi = mock(ApplicationApi.class); when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("tenant:app:default")); @@ -179,6 +183,8 @@ public class HostedVespaPolicyTest { InOrder order = inOrder(applicationApi, clusterPolicy, storageNode1, storageNode3); OrchestratorContext context = mock(OrchestratorContext.class); + OrchestratorContext probeContext = mock(OrchestratorContext.class); + when(context.createSubcontextForSingleAppOp(true)).thenReturn(probeContext); policy.acquirePermissionToRemove(context, applicationApi); order.verify(applicationApi).getClusters(); @@ -187,8 +193,8 @@ public class HostedVespaPolicyTest { order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi3); order.verify(applicationApi).getStorageNodesInGroupInClusterOrder(); - order.verify(storageNode1).setNodeState(context, ClusterControllerNodeState.DOWN); - order.verify(storageNode3).setNodeState(context, ClusterControllerNodeState.DOWN); + order.verify(storageNode1).setStorageNodeState(probeContext, ClusterControllerNodeState.DOWN); + order.verify(storageNode3).setStorageNodeState(probeContext, ClusterControllerNodeState.DOWN); order.verify(applicationApi).getNodesInGroupWith(any()); order.verify(applicationApi).setHostState(context, hostName1, HostStatus.PERMANENTLY_DOWN); diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index bf1c828e2e6..e03403f601d 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -765,11 +765,11 @@ TEST_F(ConformanceTest, testRemoveMulti) docs.push_back(testDocMan.createRandomDocumentAtLocation(0x01, i)); } - std::vector<PersistenceProvider::TimeStampAndDocumentId> ids; + std::vector<spi::IdAndTimestamp> ids; for (size_t i(0); i < docs.size(); i++) { spi->put(bucket1, Timestamp(i), docs[i]); if (i & 0x1) { - ids.emplace_back(Timestamp(i), docs[i]->getId()); + ids.emplace_back(docs[i]->getId(), Timestamp(i)); } } diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 81bfdf7f9a3..6eabadc2f86 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -476,16 +476,16 @@ DummyPersistence::updateAsync(const Bucket& bucket, Timestamp ts, DocumentUpdate } void -DummyPersistence::removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP onComplete) +DummyPersistence::removeAsync(const Bucket& b, std::vector<spi::IdAndTimestamp> ids, OperationComplete::UP onComplete) { DUMMYPERSISTENCE_VERIFY_INITIALIZED; assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); uint32_t numRemoves(0); - for (const TimeStampAndDocumentId & stampedId : ids) { - const DocumentId & id = stampedId.second; - Timestamp t = stampedId.first; + for (const spi::IdAndTimestamp & stampedId : ids) { + const DocumentId & id = stampedId.id; + Timestamp t = stampedId.timestamp; LOG(debug, "remove(%s, %" PRIu64 ", %s)", b.toString().c_str(), uint64_t(t), id.toString().c_str()); while (!bc) { diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index e015185e5b0..56602b3ab00 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -160,7 +160,7 @@ public: BucketInfoResult getBucketInfo(const Bucket&) const override; GetResult get(const Bucket&, const document::FieldSet&, const DocumentId&, Context&) const override; void putAsync(const Bucket&, Timestamp, DocumentSP, OperationComplete::UP) override; - void removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP) override; + void removeAsync(const Bucket& b, std::vector<spi::IdAndTimestamp> ids, OperationComplete::UP) override; void updateAsync(const Bucket&, Timestamp, DocumentUpdateSP, OperationComplete::UP) override; CreateIteratorResult diff --git a/persistence/src/vespa/persistence/spi/CMakeLists.txt b/persistence/src/vespa/persistence/spi/CMakeLists.txt index e4bae1c7551..bc94020ae95 100644 --- a/persistence/src/vespa/persistence/spi/CMakeLists.txt +++ b/persistence/src/vespa/persistence/spi/CMakeLists.txt @@ -10,6 +10,7 @@ vespa_add_library(persistence_spi OBJECT context.cpp docentry.cpp exceptions.cpp + id_and_timestamp.cpp persistenceprovider.cpp read_consistency.cpp resource_usage.cpp diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp index f301a9c5428..04d06235f59 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp @@ -10,8 +10,8 @@ void AbstractPersistenceProvider::removeIfFoundAsync(const Bucket& b, Timestamp timestamp, const DocumentId& id, OperationComplete::UP onComplete) { - std::vector<TimeStampAndDocumentId> ids; - ids.emplace_back(timestamp, id); + std::vector<IdAndTimestamp> ids; + ids.emplace_back(id, timestamp); removeAsync(b, std::move(ids), std::move(onComplete)); } diff --git a/persistence/src/vespa/persistence/spi/id_and_timestamp.cpp b/persistence/src/vespa/persistence/spi/id_and_timestamp.cpp new file mode 100644 index 00000000000..fba45990744 --- /dev/null +++ b/persistence/src/vespa/persistence/spi/id_and_timestamp.cpp @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "id_and_timestamp.h" + +namespace storage::spi { + +IdAndTimestamp::IdAndTimestamp() : id(), timestamp(0) {} +IdAndTimestamp::IdAndTimestamp(document::DocumentId id_, Timestamp timestamp_) noexcept + : id(std::move(id_)), + timestamp(timestamp_) +{} + +IdAndTimestamp::IdAndTimestamp(const IdAndTimestamp&) = default; +IdAndTimestamp& IdAndTimestamp::operator=(const IdAndTimestamp&) = default; +IdAndTimestamp::IdAndTimestamp(IdAndTimestamp&&) noexcept = default; +IdAndTimestamp& IdAndTimestamp::operator=(IdAndTimestamp&&) noexcept = default; + +} diff --git a/persistence/src/vespa/persistence/spi/id_and_timestamp.h b/persistence/src/vespa/persistence/spi/id_and_timestamp.h new file mode 100644 index 00000000000..d8cdba3d063 --- /dev/null +++ b/persistence/src/vespa/persistence/spi/id_and_timestamp.h @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "types.h" +#include <vespa/document/base/documentid.h> + +namespace storage::spi { + +/** + * Convenience wrapper for referencing a document ID at a particular timestamp. + * + * Prefer this instead of a std::pair due to named fields and a pre-provided hash function. + */ +struct IdAndTimestamp { + document::DocumentId id; + Timestamp timestamp; + + IdAndTimestamp(); + IdAndTimestamp(document::DocumentId id_, Timestamp timestamp_) noexcept; + + IdAndTimestamp(const IdAndTimestamp&); + IdAndTimestamp& operator=(const IdAndTimestamp&); + IdAndTimestamp(IdAndTimestamp&&) noexcept; + IdAndTimestamp& operator=(IdAndTimestamp&&) noexcept; + + bool operator==(const IdAndTimestamp& rhs) const noexcept { + return ((id == rhs.id) && (timestamp == rhs.timestamp)); + } + + struct hash { + size_t operator()(const IdAndTimestamp& id_ts) const noexcept { + const size_t h = document::GlobalId::hash()(id_ts.id.getGlobalId()); + return h ^ (id_ts.timestamp + 0x9e3779b9U + (h << 6U) + (h >> 2U)); // Basically boost::hash_combine + } + }; +}; + +} diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp index 03cefb8df89..911b3753b1f 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp @@ -44,8 +44,8 @@ RemoveResult PersistenceProvider::remove(const Bucket& bucket, Timestamp timestamp, const DocumentId & docId) { auto catcher = std::make_unique<CatchResult>(); auto future = catcher->future_result(); - std::vector<TimeStampAndDocumentId> ids; - ids.emplace_back(timestamp, docId); + std::vector<IdAndTimestamp> ids; + ids.emplace_back(docId, timestamp); removeAsync(bucket, std::move(ids), std::move(catcher)); return dynamic_cast<const RemoveResult &>(*future.get()); } diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index a90d39e8334..d3e1465e528 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -4,6 +4,7 @@ #include "bucket.h" #include "bucketinfo.h" #include "context.h" +#include "id_and_timestamp.h" #include "result.h" #include "selection.h" #include "clusterstate.h" @@ -170,7 +171,7 @@ struct PersistenceProvider * @param timestamp The timestamp for the new bucket entry. * @param id The ID to remove */ - virtual void removeAsync(const Bucket&, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP) = 0; + virtual void removeAsync(const Bucket&, std::vector<IdAndTimestamp> ids, OperationComplete::UP) = 0; /** * @see remove() diff --git a/screwdriver/build-vespa.sh b/screwdriver/build-vespa.sh index ca9ada81205..6a93474c620 100755 --- a/screwdriver/build-vespa.sh +++ b/screwdriver/build-vespa.sh @@ -53,23 +53,6 @@ case $SHOULD_BUILD in esac if [[ $SHOULD_BUILD == systemtest ]]; then - dnf module enable -y ruby:2.7 - dnf install -y \ - gcc-toolset-11-annobin \ - gcc-toolset-11-annobin-plugin-gcc \ - gcc-toolset-11-binutils \ - gcc-toolset-11-gcc-c++ \ - gcc-toolset-11-libatomic-devel \ - libxml2-devel \ - ruby \ - ruby-devel \ - rubygems-devel \ - rubygem-net-telnet \ - zstd - - source /opt/rh/gcc-toolset-11/enable - gem install libxml-ruby gnuplot distribution test-unit builder concurrent-ruby bigdecimal ffi parallel - cd $HOME git clone https://github.com/vespa-engine/system-test export SYSTEM_TEST_DIR=$(pwd)/system-test diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 49fd82c3a36..fd545fcdf77 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -322,8 +322,8 @@ assertString(const std::string & exp, const std::string & fieldName, DocumentStoreAdapter &dsa, uint32_t id) { GeneralResultPtr res = getResult(dsa, id); - return EXPECT_EQUAL(exp, std::string(res->GetEntry(fieldName.c_str())->_stringval, - res->GetEntry(fieldName.c_str())->_stringlen)); + return EXPECT_EQUAL(exp, std::string(res->GetPresentEntry(fieldName.c_str())->_stringval, + res->GetPresentEntry(fieldName.c_str())->_stringlen)); } void @@ -392,24 +392,24 @@ TEST_F("requireThatAdapterHandlesAllFieldTypes", Fixture) bc.createFieldCacheRepo(f.getResultConfig())->getFieldCache("class0"), f.getMarkupFields()); GeneralResultPtr res = getResult(dsa, 0); - EXPECT_EQUAL(255u, res->GetEntry("a")->_intval); - EXPECT_EQUAL(32767u, res->GetEntry("b")->_intval); - EXPECT_EQUAL(2147483647u, res->GetEntry("c")->_intval); - EXPECT_EQUAL(2147483648u, res->GetEntry("d")->_int64val); - EXPECT_APPROX(1234.56, res->GetEntry("e")->_doubleval, 10e-5); - EXPECT_APPROX(9876.54, res->GetEntry("f")->_doubleval, 10e-5); - EXPECT_EQUAL("foo", std::string(res->GetEntry("g")->_stringval, - res->GetEntry("g")->_stringlen)); - EXPECT_EQUAL("bar", std::string(res->GetEntry("h")->_stringval, - res->GetEntry("h")->_stringlen)); - EXPECT_EQUAL("baz", std::string(res->GetEntry("i")->_dataval, - res->GetEntry("i")->_datalen)); - EXPECT_EQUAL("qux", std::string(res->GetEntry("j")->_dataval, - res->GetEntry("j")->_datalen)); - EXPECT_EQUAL("<foo>", std::string(res->GetEntry("k")->_stringval, - res->GetEntry("k")->_stringlen)); - EXPECT_EQUAL("{foo:10}", std::string(res->GetEntry("l")->_stringval, - res->GetEntry("l")->_stringlen)); + EXPECT_EQUAL(255u, res->GetPresentEntry("a")->_intval); + EXPECT_EQUAL(32767u, res->GetPresentEntry("b")->_intval); + EXPECT_EQUAL(2147483647u, res->GetPresentEntry("c")->_intval); + EXPECT_EQUAL(2147483648u, res->GetPresentEntry("d")->_int64val); + EXPECT_APPROX(1234.56, res->GetPresentEntry("e")->_doubleval, 10e-5); + EXPECT_APPROX(9876.54, res->GetPresentEntry("f")->_doubleval, 10e-5); + EXPECT_EQUAL("foo", std::string(res->GetPresentEntry("g")->_stringval, + res->GetPresentEntry("g")->_stringlen)); + EXPECT_EQUAL("bar", std::string(res->GetPresentEntry("h")->_stringval, + res->GetPresentEntry("h")->_stringlen)); + EXPECT_EQUAL("baz", std::string(res->GetPresentEntry("i")->_dataval, + res->GetPresentEntry("i")->_datalen)); + EXPECT_EQUAL("qux", std::string(res->GetPresentEntry("j")->_dataval, + res->GetPresentEntry("j")->_datalen)); + EXPECT_EQUAL("<foo>", std::string(res->GetPresentEntry("k")->_stringval, + res->GetPresentEntry("k")->_stringlen)); + EXPECT_EQUAL("{foo:10}", std::string(res->GetPresentEntry("l")->_stringval, + res->GetPresentEntry("l")->_stringlen)); } TEST_F("requireThatAdapterHandlesMultipleDocuments", Fixture) @@ -433,11 +433,11 @@ TEST_F("requireThatAdapterHandlesMultipleDocuments", Fixture) f.getMarkupFields()); { // doc 0 GeneralResultPtr res = getResult(dsa, 0); - EXPECT_EQUAL(1000u, res->GetEntry("a")->_intval); + EXPECT_EQUAL(1000u, res->GetPresentEntry("a")->_intval); } { // doc 1 GeneralResultPtr res = getResult(dsa, 1); - EXPECT_EQUAL(2000u, res->GetEntry("a")->_intval); + EXPECT_EQUAL(2000u, res->GetPresentEntry("a")->_intval); } { // doc 2 DocsumStoreValue docsum = dsa.getMappedDocsum(2); @@ -445,7 +445,7 @@ TEST_F("requireThatAdapterHandlesMultipleDocuments", Fixture) } { // doc 0 (again) GeneralResultPtr res = getResult(dsa, 0); - EXPECT_EQUAL(1000u, res->GetEntry("a")->_intval); + EXPECT_EQUAL(1000u, res->GetPresentEntry("a")->_intval); } EXPECT_EQUAL(0u, bc._str.lastSyncToken()); uint64_t flushToken = bc._str.initFlush(bc._serialNum - 1); @@ -466,8 +466,8 @@ TEST_F("requireThatAdapterHandlesDocumentIdField", Fixture) bc.createFieldCacheRepo(f.getResultConfig())->getFieldCache("class4"), f.getMarkupFields()); GeneralResultPtr res = getResult(dsa, 0); - EXPECT_EQUAL("id:ns:searchdocument::0", std::string(res->GetEntry("documentid")->_stringval, - res->GetEntry("documentid")->_stringlen)); + EXPECT_EQUAL("id:ns:searchdocument::0", std::string(res->GetPresentEntry("documentid")->_stringval, + res->GetPresentEntry("documentid")->_stringlen)); } GlobalId gid1 = DocumentId("id:ns:searchdocument::1").getGlobalId(); // lid 1 @@ -960,14 +960,14 @@ TEST_F("requireThatUrisAreUsed", Fixture) GeneralResultPtr res = getResult(dsa, 1); { vespalib::Slime slime; - decode(res->GetEntry("uriarray"), slime); + decode(res->GetPresentEntry("uriarray"), slime); EXPECT_TRUE(slime.get().valid()); EXPECT_EQUAL("http://www.example.com:82/fluke?ab=2#8", asVstring(slime.get()[0])); EXPECT_EQUAL("http://www.flickr.com:82/fluke?ab=2#9", asVstring(slime.get()[1])); } { vespalib::Slime slime; - decode(res->GetEntry("uriwset"), slime); + decode(res->GetPresentEntry("uriwset"), slime); EXPECT_TRUE(slime.get().valid()); EXPECT_EQUAL(4L, slime.get()[0]["weight"].asLong()); EXPECT_EQUAL(7L, slime.get()[1]["weight"].asLong()); @@ -1089,14 +1089,14 @@ TEST_F("requireThatRawFieldsWorks", Fixture) GeneralResultPtr res = getResult(dsa, 1); { vespalib::Slime slime; - decode(res->GetEntry("araw"), slime); + decode(res->GetPresentEntry("araw"), slime); EXPECT_TRUE(slime.get().valid()); EXPECT_EQUAL(vespalib::Base64::encode(raw1a0), b64encode(slime.get()[0])); EXPECT_EQUAL(vespalib::Base64::encode(raw1a1), b64encode(slime.get()[1])); } { vespalib::Slime slime; - decode(res->GetEntry("wraw"), slime); + decode(res->GetPresentEntry("wraw"), slime); EXPECT_TRUE(slime.get().valid()); EXPECT_EQUAL(46L, slime.get()[0]["weight"].asLong()); EXPECT_EQUAL(45L, slime.get()[1]["weight"].asLong()); diff --git a/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp b/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp index 69013e8d7c5..dcdba3b0715 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp @@ -134,8 +134,8 @@ SpiBmFeedHandler::remove(const document::Bucket& bucket, const DocumentId& docum auto provider = get_provider(bucket); if (provider) { Bucket spi_bucket(bucket); - std::vector<storage::spi::PersistenceProvider::TimeStampAndDocumentId> ids; - ids.emplace_back(Timestamp(timestamp), document_id); + std::vector<storage::spi::IdAndTimestamp> ids; + ids.emplace_back(document_id, Timestamp(timestamp)); provider->removeAsync(spi_bucket, std::move(ids), std::make_unique<MyOperationComplete>(provider, _errors, spi_bucket, tracker)); } else { ++_errors; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 425ac6aefd0..5bd92fd788f 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -95,6 +95,13 @@ SummarySetup(const vespalib::string & baseDir, const DocTypeName & docTypeName, { DocsumBlobEntryFilter docsum_blob_entry_filter; docsum_blob_entry_filter.add_skip(RES_INT); + docsum_blob_entry_filter.add_skip(RES_SHORT); + docsum_blob_entry_filter.add_skip(RES_BOOL); + docsum_blob_entry_filter.add_skip(RES_BYTE); + docsum_blob_entry_filter.add_skip(RES_FLOAT); + docsum_blob_entry_filter.add_skip(RES_DOUBLE); + docsum_blob_entry_filter.add_skip(RES_INT64); + docsum_blob_entry_filter.add_skip(RES_TENSOR); auto resultConfig = std::make_unique<ResultConfig>(docsum_blob_entry_filter); if (!resultConfig->ReadConfig(summaryCfg, make_string("SummaryManager(%s)", baseDir.c_str()).c_str())) { std::ostringstream oss; diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 5a0bcb1cd41..81a7244ba1d 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -370,21 +370,21 @@ PersistenceEngine::putAsync(const Bucket &bucket, Timestamp ts, storage::spi::Do } void -PersistenceEngine::removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP onComplete) +PersistenceEngine::removeAsync(const Bucket& b, std::vector<storage::spi::IdAndTimestamp> ids, OperationComplete::UP onComplete) { if (ids.size() == 1) { - removeAsyncSingle(b, ids[0].first, ids[0].second, std::move(onComplete)); + removeAsyncSingle(b, ids[0].timestamp, ids[0].id, std::move(onComplete)); } else { removeAsyncMulti(b, std::move(ids), std::move(onComplete)); } } void -PersistenceEngine::removeAsyncMulti(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP onComplete) { +PersistenceEngine::removeAsyncMulti(const Bucket& b, std::vector<storage::spi::IdAndTimestamp> ids, OperationComplete::UP onComplete) { ReadGuard rguard(_rwMutex); //TODO Group per document type/handler and handle in one go. - for (const TimeStampAndDocumentId & stampedId : ids) { - const document::DocumentId & id = stampedId.second; + for (const auto & stampedId : ids) { + const document::DocumentId & id = stampedId.id; if (!id.hasDocType()) { return onComplete->onComplete( std::make_unique<RemoveResult>(Result::ErrorType::PERMANENT_ERROR, @@ -399,11 +399,11 @@ PersistenceEngine::removeAsyncMulti(const Bucket& b, std::vector<TimeStampAndDoc } } auto transportContext = std::make_shared<AsyncRemoveTransportContext>(ids.size(), std::move(onComplete)); - for (const TimeStampAndDocumentId & stampedId : ids) { - const document::DocumentId & id = stampedId.second; + for (const auto & stampedId : ids) { + const document::DocumentId & id = stampedId.id; DocTypeName docType(id.getDocType()); IPersistenceHandler *handler = getHandler(rguard, b.getBucketSpace(), docType); - handler->handleRemove(feedtoken::make(transportContext), b, stampedId.first, id); + handler->handleRemove(feedtoken::make(transportContext), b, stampedId.timestamp, id); } } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index a8886e19def..c16cc6e6a83 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -89,7 +89,7 @@ private: ClusterState::SP savedClusterState(BucketSpace bucketSpace) const; std::shared_ptr<BucketExecutor> get_bucket_executor() noexcept { return _bucket_executor.lock(); } void removeAsyncSingle(const Bucket&, Timestamp, const document::DocumentId &id, OperationComplete::UP); - void removeAsyncMulti(const Bucket&, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP); + void removeAsyncMulti(const Bucket&, std::vector<storage::spi::IdAndTimestamp> ids, OperationComplete::UP); public: typedef std::unique_ptr<PersistenceEngine> UP; @@ -107,7 +107,7 @@ public: void setActiveStateAsync(const Bucket&, BucketInfo::ActiveState, OperationComplete::UP) override; BucketInfoResult getBucketInfo(const Bucket&) const override; void putAsync(const Bucket &, Timestamp, storage::spi::DocumentSP, OperationComplete::UP) override; - void removeAsync(const Bucket&, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP) override; + void removeAsync(const Bucket&, std::vector<storage::spi::IdAndTimestamp> ids, OperationComplete::UP) override; void updateAsync(const Bucket&, Timestamp, storage::spi::DocumentUpdateSP, OperationComplete::UP) override; GetResult get(const Bucket&, const document::FieldSet&, const document::DocumentId&, Context&) const override; CreateIteratorResult diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 43a8da19191..f9b6271b47b 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -46,6 +46,7 @@ vespa_define_module( src/vespa/searchlib/test src/vespa/searchlib/test/diskindex src/vespa/searchlib/test/fakedata + src/vespa/searchlib/test/features src/vespa/searchlib/test/memoryindex src/vespa/searchlib/transactionlog src/vespa/searchlib/uca diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index b6953ec5dca..b93398e16a1 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -46,8 +46,8 @@ using search::queryeval::NearestNeighborBlueprint; using search::tensor::DefaultNearestNeighborIndexFactory; using search::tensor::DenseTensorAttribute; using search::tensor::DirectTensorAttribute; +using search::tensor::DistanceCalculator; using search::tensor::DocVectorAccess; -using search::tensor::SerializedFastValueAttribute; using search::tensor::HnswIndex; using search::tensor::HnswNode; using search::tensor::NearestNeighborIndex; @@ -55,13 +55,14 @@ using search::tensor::NearestNeighborIndexFactory; using search::tensor::NearestNeighborIndexLoader; using search::tensor::NearestNeighborIndexSaver; using search::tensor::PrepareResult; +using search::tensor::SerializedFastValueAttribute; using search::tensor::TensorAttribute; using vespalib::datastore::CompactionStrategy; -using vespalib::eval::TensorSpec; using vespalib::eval::CellType; -using vespalib::eval::ValueType; -using vespalib::eval::Value; using vespalib::eval::SimpleValue; +using vespalib::eval::TensorSpec; +using vespalib::eval::Value; +using vespalib::eval::ValueType; using DoubleVector = std::vector<double>; using generation_t = vespalib::GenerationHandler::generation_t; @@ -1072,8 +1073,8 @@ public: search::queryeval::FieldSpec field("foo", 0, 0); auto bp = std::make_unique<NearestNeighborBlueprint>( field, - this->as_dense_tensor(), - create_query_tensor(vec_2d(17, 42)), + std::make_unique<DistanceCalculator>(this->as_dense_tensor(), + create_query_tensor(vec_2d(17, 42))), 3, approximate, 5, 100100.25, global_filter_lower_limit, 1.0); diff --git a/searchlib/src/tests/features/nns_closeness/CMakeLists.txt b/searchlib/src/tests/features/nns_closeness/CMakeLists.txt index 9a8c2d7c99f..d5f9ece096b 100644 --- a/searchlib/src/tests/features/nns_closeness/CMakeLists.txt +++ b/searchlib/src/tests/features/nns_closeness/CMakeLists.txt @@ -5,5 +5,6 @@ vespa_add_executable(searchlib_nns_closeness_test_app TEST nns_closeness_test.cpp DEPENDS searchlib + searchlib_test ) vespa_add_test(NAME searchlib_nns_closeness_test_app COMMAND searchlib_nns_closeness_test_app) diff --git a/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp b/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp index c7667b2cecd..661ee884e46 100644 --- a/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp +++ b/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp @@ -1,106 +1,24 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/searchlib/features/setup.h> -#include <vespa/searchlib/fef/test/indexenvironment.h> -#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> -#include <vespa/searchlib/fef/test/queryenvironment.h> -#include <vespa/searchlib/fef/test/labels.h> #include <vespa/searchlib/features/closenessfeature.h> -#include <vespa/searchlib/fef/fef.h> +#include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/searchlib/fef/test/labels.h> +#include <vespa/searchlib/test/features/distance_closeness_fixture.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> using search::feature_t; -using namespace search::fef; -using namespace search::fef::test; +using namespace search::features::test; using namespace search::features; -using CollectionType = FieldInfo::CollectionType; -using DataType = FieldInfo::DataType; +using namespace search::fef::test; +using namespace search::fef; const vespalib::string labelFeatureName("closeness(label,nns)"); const vespalib::string fieldFeatureName("closeness(bar)"); -struct BlueprintFactoryFixture { - BlueprintFactory factory; - BlueprintFactoryFixture() : factory() - { - setup_search_features(factory); - } -}; - -struct IndexFixture { - IndexEnvironment indexEnv; - IndexFixture() : indexEnv() - { - IndexEnvironmentBuilder builder(indexEnv); - builder.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "foo"); - builder.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::TENSOR, "bar"); - } -}; - -struct FeatureDumpFixture : public IDumpFeatureVisitor { - virtual void visitDumpFeature(const vespalib::string &) override { - TEST_ERROR("no features should be dumped"); - } - FeatureDumpFixture() : IDumpFeatureVisitor() {} - ~FeatureDumpFixture() override; -}; - -FeatureDumpFixture::~FeatureDumpFixture() = default; - -struct RankFixture : BlueprintFactoryFixture, IndexFixture { - QueryEnvironment queryEnv; - RankSetup rankSetup; - MatchDataLayout mdl; - MatchData::UP match_data; - RankProgram::UP rankProgram; - std::vector<TermFieldHandle> fooHandles; - std::vector<TermFieldHandle> barHandles; - RankFixture(size_t fooCnt, size_t barCnt, const Labels &labels, const vespalib::string &featureName) - : queryEnv(&indexEnv), rankSetup(factory, indexEnv), - mdl(), match_data(), rankProgram(), fooHandles(), barHandles() - { - for (size_t i = 0; i < fooCnt; ++i) { - uint32_t fieldId = indexEnv.getFieldByName("foo")->id(); - fooHandles.push_back(mdl.allocTermField(fieldId)); - SimpleTermData term; - term.setUniqueId(i + 1); - term.addField(fieldId).setHandle(fooHandles.back()); - queryEnv.getTerms().push_back(term); - } - for (size_t i = 0; i < barCnt; ++i) { - uint32_t fieldId = indexEnv.getFieldByName("bar")->id(); - barHandles.push_back(mdl.allocTermField(fieldId)); - SimpleTermData term; - term.setUniqueId(fooCnt + i + 1); - term.addField(fieldId).setHandle(barHandles.back()); - queryEnv.getTerms().push_back(term); - } - labels.inject(queryEnv.getProperties()); - rankSetup.setFirstPhaseRank(featureName); - rankSetup.setIgnoreDefaultRankFeatures(true); - ASSERT_TRUE(rankSetup.compile()); - match_data = mdl.createMatchData(); - rankProgram = rankSetup.create_first_phase_program(); - rankProgram->setup(*match_data, queryEnv); - } - feature_t getScore(uint32_t docId) { - return Utils::getScoreFeature(*rankProgram, docId); - } - void setScore(TermFieldHandle handle, uint32_t docId, feature_t score) { - match_data->resolveTermField(handle)->setRawScore(docId, score); - } - void setFooScore(uint32_t i, uint32_t docId, feature_t distance) { - ASSERT_LESS(i, fooHandles.size()); - setScore(fooHandles[i], docId, 1.0/(1.0+distance)); - } - void setBarScore(uint32_t i, uint32_t docId, feature_t distance) { - ASSERT_LESS(i, barHandles.size()); - setScore(barHandles[i], docId, 1.0/(1.0+distance)); - } -}; +using RankFixture = DistanceClosenessFixture; TEST_F("require that blueprint can be created from factory", BlueprintFactoryFixture) { Blueprint::SP bp = f.factory.createBlueprint("closeness"); @@ -108,11 +26,11 @@ TEST_F("require that blueprint can be created from factory", BlueprintFactoryFix EXPECT_TRUE(dynamic_cast<ClosenessBlueprint*>(bp.get()) != 0); } -TEST_FFF("require that no features are dumped", ClosenessBlueprint, IndexFixture, FeatureDumpFixture) { +TEST_FFF("require that no features are dumped", ClosenessBlueprint, IndexEnvironmentFixture, FeatureDumpFixture) { f1.visitDumpFeatures(f2.indexEnv, f3); } -TEST_FF("require that setup can be done on random label", ClosenessBlueprint, IndexFixture) { +TEST_FF("require that setup can be done on random label", ClosenessBlueprint, IndexEnvironmentFixture) { DummyDependencyHandler deps(f1); f1.setName(vespalib::make_string("%s(label,random_label)", f1.getBaseName().c_str())); EXPECT_TRUE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>{"label", "random_label"})); diff --git a/searchlib/src/tests/features/nns_distance/CMakeLists.txt b/searchlib/src/tests/features/nns_distance/CMakeLists.txt index 8dbee37a194..bf4e533ea45 100644 --- a/searchlib/src/tests/features/nns_distance/CMakeLists.txt +++ b/searchlib/src/tests/features/nns_distance/CMakeLists.txt @@ -5,5 +5,6 @@ vespa_add_executable(searchlib_nns_distance_test_app TEST nns_distance_test.cpp DEPENDS searchlib + searchlib_test ) vespa_add_test(NAME searchlib_nns_distance_test_app COMMAND searchlib_nns_distance_test_app) diff --git a/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp b/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp index 1e81b5576c1..6b2669367ad 100644 --- a/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp +++ b/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp @@ -2,105 +2,23 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/features/setup.h> -#include <vespa/searchlib/fef/test/indexenvironment.h> -#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> -#include <vespa/searchlib/fef/test/queryenvironment.h> #include <vespa/searchlib/fef/test/labels.h> #include <vespa/searchlib/features/distancefeature.h> -#include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/searchlib/test/features/distance_closeness_fixture.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> using search::feature_t; -using namespace search::fef; -using namespace search::fef::test; +using namespace search::features::test; using namespace search::features; -using CollectionType = FieldInfo::CollectionType; -using DataType = FieldInfo::DataType; +using namespace search::fef::test; +using namespace search::fef; -const vespalib::string labelFeatureName("distance(label,label)"); +const vespalib::string labelFeatureName("distance(label,nns)"); const vespalib::string fieldFeatureName("distance(bar)"); -struct BlueprintFactoryFixture { - BlueprintFactory factory; - BlueprintFactoryFixture() : factory() - { - setup_search_features(factory); - } -}; - -struct IndexFixture { - IndexEnvironment indexEnv; - IndexFixture() : indexEnv() - { - IndexEnvironmentBuilder builder(indexEnv); - builder.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "foo"); - builder.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::TENSOR, "bar"); - } -}; - -struct FeatureDumpFixture : public IDumpFeatureVisitor { - virtual void visitDumpFeature(const vespalib::string &) override { - TEST_ERROR("no features should be dumped"); - } - FeatureDumpFixture() : IDumpFeatureVisitor() {} - ~FeatureDumpFixture() override; -}; - -FeatureDumpFixture::~FeatureDumpFixture() = default; - -struct RankFixture : BlueprintFactoryFixture, IndexFixture { - QueryEnvironment queryEnv; - RankSetup rankSetup; - MatchDataLayout mdl; - MatchData::UP match_data; - RankProgram::UP rankProgram; - std::vector<TermFieldHandle> fooHandles; - std::vector<TermFieldHandle> barHandles; - RankFixture(size_t fooCnt, size_t barCnt, const Labels &labels, const vespalib::string &featureName) - : queryEnv(&indexEnv), rankSetup(factory, indexEnv), - mdl(), match_data(), rankProgram(), fooHandles(), barHandles() - { - for (size_t i = 0; i < fooCnt; ++i) { - uint32_t fieldId = indexEnv.getFieldByName("foo")->id(); - fooHandles.push_back(mdl.allocTermField(fieldId)); - SimpleTermData term; - term.setUniqueId(i + 1); - term.addField(fieldId).setHandle(fooHandles.back()); - queryEnv.getTerms().push_back(term); - } - for (size_t i = 0; i < barCnt; ++i) { - uint32_t fieldId = indexEnv.getFieldByName("bar")->id(); - barHandles.push_back(mdl.allocTermField(fieldId)); - SimpleTermData term; - term.setUniqueId(fooCnt + i + 1); - term.addField(fieldId).setHandle(barHandles.back()); - queryEnv.getTerms().push_back(term); - } - labels.inject(queryEnv.getProperties()); - rankSetup.setFirstPhaseRank(featureName); - rankSetup.setIgnoreDefaultRankFeatures(true); - ASSERT_TRUE(rankSetup.compile()); - match_data = mdl.createMatchData(); - rankProgram = rankSetup.create_first_phase_program(); - rankProgram->setup(*match_data, queryEnv); - } - feature_t getScore(uint32_t docId) { - return Utils::getScoreFeature(*rankProgram, docId); - } - void setScore(TermFieldHandle handle, uint32_t docId, feature_t score) { - match_data->resolveTermField(handle)->setRawScore(docId, score); - } - void setFooScore(uint32_t i, uint32_t docId, feature_t distance) { - ASSERT_LESS(i, fooHandles.size()); - setScore(fooHandles[i], docId, 1.0/(1.0+distance)); - } - void setBarScore(uint32_t i, uint32_t docId, feature_t distance) { - ASSERT_LESS(i, barHandles.size()); - setScore(barHandles[i], docId, 1.0/(1.0+distance)); - } -}; +using RankFixture = DistanceClosenessFixture; TEST_F("require that blueprint can be created from factory", BlueprintFactoryFixture) { Blueprint::SP bp = f.factory.createBlueprint("distance"); @@ -108,17 +26,17 @@ TEST_F("require that blueprint can be created from factory", BlueprintFactoryFix EXPECT_TRUE(dynamic_cast<DistanceBlueprint*>(bp.get()) != 0); } -TEST_FFF("require that no features are dumped", DistanceBlueprint, IndexFixture, FeatureDumpFixture) { +TEST_FFF("require that no features are dumped", DistanceBlueprint, IndexEnvironmentFixture, FeatureDumpFixture) { f1.visitDumpFeatures(f2.indexEnv, f3); } -TEST_FF("require that setup can be done on random label", DistanceBlueprint, IndexFixture) { +TEST_FF("require that setup can be done on random label", DistanceBlueprint, IndexEnvironmentFixture) { DummyDependencyHandler deps(f1); f1.setName(vespalib::make_string("%s(label,random_label)", f1.getBaseName().c_str())); EXPECT_TRUE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>{"label", "random_label"})); } -TEST_FF("require that setup with unknown field fails", DistanceBlueprint, IndexFixture) { +TEST_FF("require that setup with unknown field fails", DistanceBlueprint, IndexEnvironmentFixture) { DummyDependencyHandler deps(f1); f1.setName(vespalib::make_string("%s(field,random_fieldname)", f1.getBaseName().c_str())); EXPECT_FALSE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>{"field", "random_fieldname"})); @@ -132,7 +50,7 @@ TEST_FF("require that unrelated label gives max-double distance", SingleLabel("u EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10)); } -TEST_FF("require that labeled item raw score can be obtained", SingleLabel("label", 1), RankFixture(2, 2, f1, labelFeatureName)) { +TEST_FF("require that labeled item raw score can be obtained", SingleLabel("nns", 1), RankFixture(2, 2, f1, labelFeatureName)) { f2.setFooScore(0, 10, 5.0); EXPECT_EQUAL(5.0, f2.getScore(10)); } @@ -142,7 +60,7 @@ TEST_FF("require that field raw score can be obtained", NoLabel(), RankFixture(2 EXPECT_EQUAL(5.0, f2.getScore(10)); } -TEST_FF("require that other raw scores are ignored", SingleLabel("label", 2), RankFixture(2, 2, f1, labelFeatureName)) { +TEST_FF("require that other raw scores are ignored", SingleLabel("nns", 2), RankFixture(2, 2, f1, labelFeatureName)) { f2.setFooScore(0, 10, 1.0); f2.setFooScore(1, 10, 2.0); f2.setBarScore(0, 10, 5.0); @@ -158,7 +76,7 @@ TEST_FF("require that the correct raw score is used", NoLabel(), RankFixture(2, EXPECT_EQUAL(7.0, f2.getScore(10)); } -TEST_FF("require that stale data is ignored", SingleLabel("label", 2), RankFixture(2, 2, f1, labelFeatureName)) { +TEST_FF("require that stale data is ignored", SingleLabel("nns", 2), RankFixture(2, 2, f1, labelFeatureName)) { f2.setFooScore(0, 10, 1.0); f2.setFooScore(1, 5, 2.0); EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10)); diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index d9db50ae816..7af2186ed1e 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -599,13 +599,6 @@ bool check_valid_diversity_attr(const IAttributeVector *attr) { return (attr->hasEnum() || attr->isIntegerType() || attr->isFloatingPointType()); } -bool -is_compatible_for_nearest_neighbor(const vespalib::eval::ValueType& lhs, - const vespalib::eval::ValueType& rhs) -{ - return (lhs.dimensions() == rhs.dimensions()); -} - //----------------------------------------------------------------------------- @@ -760,40 +753,24 @@ public: setResult(std::make_unique<queryeval::EmptyBlueprint>(_field)); } void visit(query::NearestNeighborTerm &n) override { - const ITensorAttribute *tensor_attr = _attr.asTensorAttribute(); - if (tensor_attr == nullptr) { - return fail_nearest_neighbor_term(n, "Attribute is not a tensor"); - } - const auto & ta_type = tensor_attr->getTensorType(); - if ((! ta_type.is_dense()) || (ta_type.dimensions().size() != 1)) { - return fail_nearest_neighbor_term(n, make_string("Attribute tensor type (%s) is not a dense tensor of order 1", - ta_type.to_spec().c_str())); - } const auto* query_tensor = getRequestContext().get_query_tensor(n.get_query_tensor_name()); if (query_tensor == nullptr) { return fail_nearest_neighbor_term(n, "Query tensor was not found in request context"); } - const auto & qt_type = query_tensor->type(); - if (! qt_type.is_dense()) { - return fail_nearest_neighbor_term(n, make_string("Query tensor is not a dense tensor (type=%s)", - qt_type.to_spec().c_str())); - } - if (!is_compatible_for_nearest_neighbor(ta_type, qt_type)) { - return fail_nearest_neighbor_term(n, make_string("Attribute tensor type (%s) and query tensor type (%s) are not compatible", - ta_type.to_spec().c_str(), qt_type.to_spec().c_str())); - } - if (tensor_attr->supports_extract_cells_ref() == false) { - return fail_nearest_neighbor_term(n, make_string("Attribute does not support access to tensor data (type=%s)", - ta_type.to_spec().c_str())); + try { + auto calc = tensor::DistanceCalculator::make_with_validation(_attr, *query_tensor); + setResult(std::make_unique<queryeval::NearestNeighborBlueprint>(_field, + std::move(calc), + n.get_target_num_hits(), + n.get_allow_approximate(), + n.get_explore_additional_hits(), + n.get_distance_threshold(), + getRequestContext().get_attribute_blueprint_params().global_filter_lower_limit, + getRequestContext().get_attribute_blueprint_params().global_filter_upper_limit)); + } catch (const vespalib::IllegalArgumentException& ex) { + return fail_nearest_neighbor_term(n, ex.getMessage()); + } - setResult(std::make_unique<queryeval::NearestNeighborBlueprint>(_field, *tensor_attr, - *query_tensor, - n.get_target_num_hits(), - n.get_allow_approximate(), - n.get_explore_additional_hits(), - n.get_distance_threshold(), - getRequestContext().get_attribute_blueprint_params().global_filter_lower_limit, - getRequestContext().get_attribute_blueprint_params().global_filter_upper_limit)); } void visit(query::FuzzyTerm &n) override { visitTerm(n); } diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 912c4a47c39..d3671d61ccf 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -7,6 +7,7 @@ #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/util/atomic.h> #include <vespa/fastos/types.h> +#include <algorithm> namespace vespalib { class nbostream; diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt index 88531a46cb1..8acf28f4a2f 100644 --- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt @@ -12,7 +12,7 @@ vespa_add_library(searchlib_features OBJECT debug_wait.cpp dense_tensor_attribute_executor.cpp direct_tensor_attribute_executor.cpp - great_circle_distance_feature.cpp + distance_calculator_bundle.cpp distancefeature.cpp distancetopathfeature.cpp documenttestutils.cpp @@ -29,6 +29,7 @@ vespa_add_library(searchlib_features OBJECT foreachfeature.cpp freshnessfeature.cpp global_sequence_feature.cpp + great_circle_distance_feature.cpp internal_max_reduce_prod_join_feature.cpp item_raw_score_feature.cpp jarowinklerdistancefeature.cpp diff --git a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp index 04fc2a263be..e44c94dbb2d 100644 --- a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp @@ -1,9 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "closenessfeature.h" +#include "distance_calculator_bundle.h" #include "utils.h" #include <vespa/searchcommon/common/schema.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/tensor/distance_calculator.h> #include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> @@ -16,8 +18,8 @@ namespace search::features { /** Implements the executor for converting NNS rawscore to a closeness feature. */ class ConvertRawScoreToCloseness : public fef::FeatureExecutor { private: - std::vector<fef::TermFieldHandle> _handles; - const fef::MatchData *_md; + DistanceCalculatorBundle _bundle; + const fef::MatchData *_md; void handle_bind_match_data(const fef::MatchData &md) override { _md = &md; } @@ -28,32 +30,15 @@ public: }; ConvertRawScoreToCloseness::ConvertRawScoreToCloseness(const fef::IQueryEnvironment &env, uint32_t fieldId) - : _handles(), + : _bundle(env, fieldId, "closeness"), _md(nullptr) { - _handles.reserve(env.getNumTerms()); - for (uint32_t i = 0; i < env.getNumTerms(); ++i) { - search::fef::TermFieldHandle handle = util::getTermFieldHandle(env, i, fieldId); - if (handle != search::fef::IllegalHandle) { - _handles.push_back(handle); - } - } } ConvertRawScoreToCloseness::ConvertRawScoreToCloseness(const fef::IQueryEnvironment &env, const vespalib::string &label) - : _handles(), + : _bundle(env, label, "closeness"), _md(nullptr) { - const ITermData *term = util::getTermByLabel(env, label); - if (term != nullptr) { - // expect numFields() == 1 - for (uint32_t i = 0; i < term->numFields(); ++i) { - TermFieldHandle handle = term->field(i).getHandle(); - if (handle != IllegalHandle) { - _handles.push_back(handle); - } - } - } } void @@ -61,11 +46,14 @@ ConvertRawScoreToCloseness::execute(uint32_t docId) { feature_t max_closeness = 0.0; assert(_md); - for (auto handle : _handles) { - const TermFieldMatchData *tfmd = _md->resolveTermField(handle); + for (const auto& elem : _bundle.elements()) { + const TermFieldMatchData *tfmd = _md->resolveTermField(elem.handle); if (tfmd->getDocId() == docId) { feature_t converted = tfmd->getRawScore(); max_closeness = std::max(max_closeness, converted); + } else if (elem.calc) { + feature_t converted = elem.calc->calc_raw_score(docId); + max_closeness = std::max(max_closeness, converted); } } outputs().set_number(0, max_closeness); @@ -201,15 +189,26 @@ ClosenessBlueprint::createInstance() const return std::make_unique<ClosenessBlueprint>(); } -FeatureExecutor & -ClosenessBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const +void +ClosenessBlueprint::prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const { + if (_use_nns_tensor) { + DistanceCalculatorBundle::prepare_shared_state(env, store, _attr_id, "closeness"); + } if (_use_item_label) { - return stash.create<ConvertRawScoreToCloseness>(env, _arg_string); + DistanceCalculatorBundle::prepare_shared_state(env, store, _arg_string, "closeness"); } +} + +FeatureExecutor & +ClosenessBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const +{ if (_use_nns_tensor) { return stash.create<ConvertRawScoreToCloseness>(env, _attr_id); } + if (_use_item_label) { + return stash.create<ConvertRawScoreToCloseness>(env, _arg_string); + } assert(_use_geo_pos); return stash.create<ClosenessExecutor>(_maxDistance, _scaleDistance); } diff --git a/searchlib/src/vespa/searchlib/features/closenessfeature.h b/searchlib/src/vespa/searchlib/features/closenessfeature.h index 799495eaff5..6e265e5dcb8 100644 --- a/searchlib/src/vespa/searchlib/features/closenessfeature.h +++ b/searchlib/src/vespa/searchlib/features/closenessfeature.h @@ -45,6 +45,7 @@ public: return fef::ParameterDescriptions().desc().string().desc().string().string(); } bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; + void prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const override; fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; }; diff --git a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp new file mode 100644 index 00000000000..90386dffd51 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp @@ -0,0 +1,172 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "distance_calculator_bundle.h" +#include "utils.h" +#include <vespa/searchlib/fef/iqueryenvironment.h> +#include <vespa/searchlib/fef/query_value.h> +#include <vespa/searchlib/tensor/distance_calculator.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/issue.h> + +using search::fef::ITermData; +using search::fef::IllegalHandle; +using search::fef::InvalidValueTypeException; +using search::fef::QueryValue; +using search::fef::TermFieldHandle; +using search::tensor::DistanceCalculator; +using vespalib::Issue; + +namespace search::features { + +namespace { + +void +prepare_query_tensor(const fef::IQueryEnvironment& env, + fef::IObjectStore& store, + const vespalib::string& query_tensor_name, + const vespalib::string& feature_name) +{ + try { + auto qvalue = QueryValue::from_config(query_tensor_name, env.getIndexEnvironment()); + qvalue.prepare_shared_state(env, store); + } catch (const InvalidValueTypeException& ex) { + Issue::report("%s feature: Query tensor '%s' has invalid type '%s'.", + feature_name.c_str(), query_tensor_name.c_str(), ex.type_str().c_str()); + } +} + +std::unique_ptr<DistanceCalculator> +make_distance_calculator(const fef::IQueryEnvironment& env, + const search::attribute::IAttributeVector& attr, + const vespalib::string& query_tensor_name, + const vespalib::string& feature_name) +{ + try { + auto qvalue = QueryValue::from_config(query_tensor_name, env.getIndexEnvironment()); + const auto* query_tensor = qvalue.lookup_value(env.getObjectStore()); + if (query_tensor == nullptr) { + Issue::report("%s feature: Query tensor '%s' is not found in the object store.", + feature_name.c_str(), query_tensor_name.c_str()); + return {}; + } + return DistanceCalculator::make_with_validation(attr, *query_tensor); + } catch (const InvalidValueTypeException& ex) { + Issue::report("%s feature: Query tensor '%s' has invalid type '%s'.", + feature_name.c_str(), query_tensor_name.c_str(), ex.type_str().c_str()); + } catch (const vespalib::IllegalArgumentException& ex) { + Issue::report("%s feature: Could not create distance calculator for attribute '%s' and query tensor '%s': %s.", + feature_name.c_str(), attr.getName().c_str(), query_tensor_name.c_str(), ex.getMessage().c_str()); + } + return {}; +} + +const search::attribute::IAttributeVector* +resolve_attribute_for_field(const fef::IQueryEnvironment& env, + uint32_t field_id, + const vespalib::string& feature_name) +{ + const auto* field = env.getIndexEnvironment().getField(field_id); + if (field != nullptr) { + const auto* attr = env.getAttributeContext().getAttribute(field->name()); + if (attr == nullptr) { + Issue::report("%s feature: The attribute vector '%s' for field id '%u' doesn't exist.", + feature_name.c_str(), field->name().c_str(), field_id); + } + return attr; + } + return nullptr; +} + +} + +DistanceCalculatorBundle::Element::Element(fef::TermFieldHandle handle_in) + : handle(handle_in), + calc() +{ +} + +DistanceCalculatorBundle::Element::Element(fef::TermFieldHandle handle_in, std::unique_ptr<search::tensor::DistanceCalculator> calc_in) + : handle(handle_in), + calc(std::move(calc_in)) +{ +} + +DistanceCalculatorBundle::Element::~Element() = default; + +DistanceCalculatorBundle::DistanceCalculatorBundle(const fef::IQueryEnvironment& env, + uint32_t field_id, + const vespalib::string& feature_name) + + : _elems() +{ + _elems.reserve(env.getNumTerms()); + const auto* attr = resolve_attribute_for_field(env, field_id, feature_name); + for (uint32_t i = 0; i < env.getNumTerms(); ++i) { + search::fef::TermFieldHandle handle = util::getTermFieldHandle(env, i, field_id); + if (handle != search::fef::IllegalHandle) { + const auto* term = env.getTerm(i); + if (term->query_tensor_name().has_value() && (attr != nullptr)) { + _elems.emplace_back(handle, make_distance_calculator(env, *attr, term->query_tensor_name().value(), feature_name)); + } else { + _elems.emplace_back(handle); + } + } + } +} + +DistanceCalculatorBundle::DistanceCalculatorBundle(const fef::IQueryEnvironment& env, + const vespalib::string& label, + const vespalib::string& feature_name) + : _elems() +{ + const ITermData* term = util::getTermByLabel(env, label); + if (term != nullptr) { + // expect numFields() == 1 + for (uint32_t i = 0; i < term->numFields(); ++i) { + const auto& term_field = term->field(i); + TermFieldHandle handle = term_field.getHandle(); + if (handle != IllegalHandle) { + std::unique_ptr<DistanceCalculator> calc; + if (term->query_tensor_name().has_value()) { + const auto* attr = resolve_attribute_for_field(env, term_field.getFieldId(), feature_name); + if (attr != nullptr) { + calc = make_distance_calculator(env, *attr, term->query_tensor_name().value(), feature_name); + } + } + _elems.emplace_back(handle, std::move(calc)); + } + } + } +} + +void +DistanceCalculatorBundle::prepare_shared_state(const fef::IQueryEnvironment& env, + fef::IObjectStore& store, + uint32_t field_id, + const vespalib::string& feature_name) +{ + for (uint32_t i = 0; i < env.getNumTerms(); ++i) { + search::fef::TermFieldHandle handle = util::getTermFieldHandle(env, i, field_id); + if (handle != search::fef::IllegalHandle) { + const auto* term = env.getTerm(i); + if (term->query_tensor_name().has_value()) { + prepare_query_tensor(env, store, term->query_tensor_name().value(), feature_name); + } + } + } +} + +void +DistanceCalculatorBundle::prepare_shared_state(const fef::IQueryEnvironment& env, + fef::IObjectStore& store, + const vespalib::string& label, + const vespalib::string& feature_name) +{ + const auto* term = util::getTermByLabel(env, label); + if ((term != nullptr) && term->query_tensor_name().has_value()) { + prepare_query_tensor(env, store, term->query_tensor_name().value(), feature_name); + } +} + +} + diff --git a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h new file mode 100644 index 00000000000..dd3fc521d96 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h @@ -0,0 +1,59 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/fef/handle.h> +#include <vespa/vespalib/stllike/string.h> +#include <memory> +#include <vector> + +namespace search::tensor { class DistanceCalculator; } +namespace search::fef { +class IObjectStore; +class IQueryEnvironment; +} + +namespace search::features { + +/** + * A bundle of term-field tuples (TermFieldHandle, DistanceCalculator) used by the closeness and distance rank features. + * + * For most document ids the raw score is available in the TermFieldMatchData retrieved using the TermFieldHandle, + * as it was calculated during matching. In the other cases the DistanceCalculator can be used to calculate the score on the fly. + */ +class DistanceCalculatorBundle { +public: + struct Element { + fef::TermFieldHandle handle; + std::unique_ptr<search::tensor::DistanceCalculator> calc; + Element(Element&& rhs) noexcept = default; // Needed as std::vector::reserve() is used. + Element(fef::TermFieldHandle handle_in); + Element(fef::TermFieldHandle handle_in, std::unique_ptr<search::tensor::DistanceCalculator> calc_in); + ~Element(); + }; +private: + std::vector<Element> _elems; + +public: + DistanceCalculatorBundle(const fef::IQueryEnvironment& env, + uint32_t field_id, + const vespalib::string& feature_name); + + DistanceCalculatorBundle(const fef::IQueryEnvironment& env, + const vespalib::string& label, + const vespalib::string& feature_name); + + const std::vector<Element>& elements() const { return _elems; } + + static void prepare_shared_state(const fef::IQueryEnvironment& env, + fef::IObjectStore& store, + uint32_t field_id, + const vespalib::string& feature_name); + + static void prepare_shared_state(const fef::IQueryEnvironment& env, + fef::IObjectStore& store, + const vespalib::string& label, + const vespalib::string& feature_name); +}; + +} diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index d0a2c1a3838..6add65054ac 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -1,16 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "distance_calculator_bundle.h" #include "distancefeature.h" +#include "utils.h" +#include <vespa/document/datatype/positiondatatype.h> #include <vespa/searchcommon/common/schema.h> #include <vespa/searchlib/common/geo_location_spec.h> #include <vespa/searchlib/fef/matchdata.h> -#include <vespa/document/datatype/positiondatatype.h> +#include <vespa/searchlib/tensor/distance_calculator.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/util/issue.h> #include <vespa/vespalib/util/stash.h> #include <cmath> #include <limits> -#include "utils.h" #include <vespa/log/log.h> LOG_SETUP(".features.distancefeature"); @@ -24,8 +26,8 @@ namespace search::features { /** Implements the executor for converting NNS rawscore to a distance feature. */ class ConvertRawscoreToDistance : public fef::FeatureExecutor { private: - std::vector<fef::TermFieldHandle> _handles; - const fef::MatchData *_md; + DistanceCalculatorBundle _bundle; + const fef::MatchData *_md; void handle_bind_match_data(const fef::MatchData &md) override { _md = &md; } @@ -36,32 +38,15 @@ public: }; ConvertRawscoreToDistance::ConvertRawscoreToDistance(const fef::IQueryEnvironment &env, uint32_t fieldId) - : _handles(), + : _bundle(env, fieldId, "distance"), _md(nullptr) { - _handles.reserve(env.getNumTerms()); - for (uint32_t i = 0; i < env.getNumTerms(); ++i) { - search::fef::TermFieldHandle handle = util::getTermFieldHandle(env, i, fieldId); - if (handle != search::fef::IllegalHandle) { - _handles.push_back(handle); - } - } } ConvertRawscoreToDistance::ConvertRawscoreToDistance(const fef::IQueryEnvironment &env, const vespalib::string &label) - : _handles(), + : _bundle(env, label, "distance"), _md(nullptr) { - const ITermData *term = util::getTermByLabel(env, label); - if (term != nullptr) { - // expect numFields() == 1 - for (uint32_t i = 0; i < term->numFields(); ++i) { - TermFieldHandle handle = term->field(i).getHandle(); - if (handle != IllegalHandle) { - _handles.push_back(handle); - } - } - } } void @@ -69,12 +54,16 @@ ConvertRawscoreToDistance::execute(uint32_t docId) { feature_t min_distance = std::numeric_limits<feature_t>::max(); assert(_md); - for (auto handle : _handles) { - const TermFieldMatchData *tfmd = _md->resolveTermField(handle); + for (const auto& elem : _bundle.elements()) { + const TermFieldMatchData *tfmd = _md->resolveTermField(elem.handle); if (tfmd->getDocId() == docId) { feature_t invdist = tfmd->getRawScore(); feature_t converted = (1.0 / invdist) - 1.0; min_distance = std::min(min_distance, converted); + } else if (elem.calc) { + feature_t invdist = elem.calc->calc_raw_score(docId); + feature_t converted = (1.0 / invdist) - 1.0; + min_distance = std::min(min_distance, converted); } } outputs().set_number(0, min_distance); @@ -249,6 +238,17 @@ DistanceBlueprint::setup(const IIndexEnvironment & env, return false; } +void +DistanceBlueprint::prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const +{ + if (_use_nns_tensor) { + DistanceCalculatorBundle::prepare_shared_state(env, store, _attr_id, "distance"); + } + if (_use_item_label) { + DistanceCalculatorBundle::prepare_shared_state(env, store, _arg_string, "distance"); + } +} + FeatureExecutor & DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const { diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.h b/searchlib/src/vespa/searchlib/features/distancefeature.h index 6eff0380c3a..bf578d45f42 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.h +++ b/searchlib/src/vespa/searchlib/features/distancefeature.h @@ -63,6 +63,7 @@ public: return fef::ParameterDescriptions().desc().string().desc().string().string(); } bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; + void prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const override; fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; }; diff --git a/searchlib/src/vespa/searchlib/fef/query_value.cpp b/searchlib/src/vespa/searchlib/fef/query_value.cpp index d9cdb0aa23d..a60a24425b5 100644 --- a/searchlib/src/vespa/searchlib/fef/query_value.cpp +++ b/searchlib/src/vespa/searchlib/fef/query_value.cpp @@ -161,6 +161,8 @@ QueryValue::QueryValue(const vespalib::string& key, const vespalib::eval::ValueT { } +QueryValue::~QueryValue() = default; + QueryValue QueryValue::from_config(const vespalib::string& key, const IIndexEnvironment& env) { diff --git a/searchlib/src/vespa/searchlib/fef/query_value.h b/searchlib/src/vespa/searchlib/fef/query_value.h index 477b6aa451f..3cdb90ea871 100644 --- a/searchlib/src/vespa/searchlib/fef/query_value.h +++ b/searchlib/src/vespa/searchlib/fef/query_value.h @@ -59,6 +59,7 @@ private: public: QueryValue(); QueryValue(const vespalib::string& key, const vespalib::eval::ValueType& type); + ~QueryValue(); /** * Create a QueryValue using properties from the given index environment to extract the value type. diff --git a/searchlib/src/vespa/searchlib/fef/test/labels.h b/searchlib/src/vespa/searchlib/fef/test/labels.h index 5f27f786405..a69634003b4 100644 --- a/searchlib/src/vespa/searchlib/fef/test/labels.h +++ b/searchlib/src/vespa/searchlib/fef/test/labels.h @@ -1,5 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index a36a0006c76..6a891341afd 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -34,8 +34,7 @@ to_string(NearestNeighborBlueprint::Algorithm algorithm) } // namespace <unnamed> NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& field, - const tensor::ITensorAttribute& attr_tensor, - const Value& query_tensor, + std::unique_ptr<search::tensor::DistanceCalculator> distance_calc, uint32_t target_hits, bool approximate, uint32_t explore_additional_hits, @@ -43,9 +42,9 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f double global_filter_lower_limit, double global_filter_upper_limit) : ComplexLeafBlueprint(field), - _attr_tensor(attr_tensor), - _distance_calc(_attr_tensor, query_tensor), - _query_tensor(_distance_calc.query_tensor()), + _distance_calc(std::move(distance_calc)), + _attr_tensor(_distance_calc->attribute_tensor()), + _query_tensor(_distance_calc->query_tensor()), _target_hits(target_hits), _adjusted_target_hits(target_hits), _approximate(approximate), @@ -62,7 +61,7 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _global_filter_hit_ratio() { if (distance_threshold < std::numeric_limits<double>::max()) { - _distance_threshold = _distance_calc.function().convert_threshold(distance_threshold); + _distance_threshold = _distance_calc->function().convert_threshold(distance_threshold); _distance_heap.set_distance_threshold(_distance_threshold); } uint32_t est_hits = _attr_tensor.get_num_docs(); @@ -127,11 +126,11 @@ NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchData switch (_algorithm) { case Algorithm::INDEX_TOP_K_WITH_FILTER: case Algorithm::INDEX_TOP_K: - return NnsIndexIterator::create(tfmd, _found_hits, _distance_calc.function()); + return NnsIndexIterator::create(tfmd, _found_hits, _distance_calc->function()); default: ; } - return NearestNeighborIterator::create(strict, tfmd, _distance_calc, + return NearestNeighborIterator::create(strict, tfmd, *_distance_calc, _distance_heap, _global_filter->filter()); } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h index 9948cce1407..3dd03291b97 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h @@ -28,8 +28,8 @@ public: INDEX_TOP_K_WITH_FILTER }; private: + std::unique_ptr<search::tensor::DistanceCalculator> _distance_calc; const tensor::ITensorAttribute& _attr_tensor; - search::tensor::DistanceCalculator _distance_calc; const vespalib::eval::Value& _query_tensor; uint32_t _target_hits; uint32_t _adjusted_target_hits; @@ -49,8 +49,7 @@ private: void perform_top_k(const search::tensor::NearestNeighborIndex* nns_index); public: NearestNeighborBlueprint(const queryeval::FieldSpec& field, - const tensor::ITensorAttribute& attr_tensor, - const vespalib::eval::Value& query_tensor, + std::unique_ptr<search::tensor::DistanceCalculator> distance_calc, uint32_t target_hits, bool approximate, uint32_t explore_additional_hits, double distance_threshold, double global_filter_lower_limit, diff --git a/searchlib/src/vespa/searchlib/tensor/distance_calculator.cpp b/searchlib/src/vespa/searchlib/tensor/distance_calculator.cpp index adfa5b7ee4a..d6d5433ff15 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_calculator.cpp +++ b/searchlib/src/vespa/searchlib/tensor/distance_calculator.cpp @@ -4,12 +4,17 @@ #include "distance_function_factory.h" #include "nearest_neighbor_index.h" #include <vespa/eval/eval/fast_value.h> +#include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> +using vespalib::IllegalArgumentException; using vespalib::eval::CellType; using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::TypedCells; using vespalib::eval::Value; using vespalib::eval::ValueType; +using vespalib::make_string; namespace { @@ -42,6 +47,13 @@ struct ConvertCellsSelector } }; +bool +is_compatible(const vespalib::eval::ValueType& lhs, + const vespalib::eval::ValueType& rhs) +{ + return (lhs.dimensions() == rhs.dimensions()); +} + } namespace search::tensor { @@ -86,5 +98,40 @@ DistanceCalculator::DistanceCalculator(const tensor::ITensorAttribute& attr_tens DistanceCalculator::~DistanceCalculator() = default; +namespace { + + + +} + +std::unique_ptr<DistanceCalculator> +DistanceCalculator::make_with_validation(const search::attribute::IAttributeVector& attr, + const vespalib::eval::Value& query_tensor_in) +{ + const ITensorAttribute* attr_tensor = attr.asTensorAttribute(); + if (attr_tensor == nullptr) { + throw IllegalArgumentException("Attribute is not a tensor"); + } + const auto& at_type = attr_tensor->getTensorType(); + if ((!at_type.is_dense()) || (at_type.dimensions().size() != 1)) { + throw IllegalArgumentException(make_string("Attribute tensor type (%s) is not a dense tensor of order 1", + at_type.to_spec().c_str())); + } + const auto& qt_type = query_tensor_in.type(); + if (!qt_type.is_dense()) { + throw IllegalArgumentException(make_string("Query tensor type (%s) is not a dense tensor", + qt_type.to_spec().c_str())); + } + if (!is_compatible(at_type, qt_type)) { + throw IllegalArgumentException(make_string("Attribute tensor type (%s) and query tensor type (%s) are not compatible", + at_type.to_spec().c_str(), qt_type.to_spec().c_str())); + } + if (!attr_tensor->supports_extract_cells_ref()) { + throw IllegalArgumentException(make_string("Attribute tensor does not support access to tensor data (type=%s)", + at_type.to_spec().c_str())); + } + return std::make_unique<DistanceCalculator>(*attr_tensor, query_tensor_in); +} + } diff --git a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h index f1cc7feb9df..3ef41906b92 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h @@ -6,6 +6,8 @@ namespace vespalib::eval { struct Value; } +namespace search::attribute { class IAttributeVector; } + namespace search::tensor { /** @@ -40,9 +42,23 @@ public: const vespalib::eval::Value& query_tensor() const { return *_query_tensor; } const DistanceFunction& function() const { return *_dist_fun; } + double calc_raw_score(uint32_t docid) const { + double distance = _dist_fun->calc(_query_tensor_cells, _attr_tensor.extract_cells_ref(docid)); + return _dist_fun->to_rawscore(distance); + } + double calc_with_limit(uint32_t docid, double limit) const { return _dist_fun->calc_with_limit(_query_tensor_cells, _attr_tensor.extract_cells_ref(docid), limit); } + + /** + * Create a calculator for the given attribute tensor and query tensor, if possible. + * + * Throws vespalib::IllegalArgumentException if the inputs are not supported or incompatible. + */ + static std::unique_ptr<DistanceCalculator> make_with_validation(const search::attribute::IAttributeVector& attr, + const vespalib::eval::Value& query_tensor_in); + }; } diff --git a/searchlib/src/vespa/searchlib/test/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/CMakeLists.txt index 0d56fb8b4be..ed884a46217 100644 --- a/searchlib/src/vespa/searchlib/test/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/CMakeLists.txt @@ -13,6 +13,7 @@ vespa_add_library(searchlib_test $<TARGET_OBJECTS:searchlib_test_gtest_migration> DEPENDS searchlib + searchlib_searchlib_test_features searchlib_searchlib_test_memoryindex GTest::GTest ) diff --git a/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt new file mode 100644 index 00000000000..ba70fe57e88 --- /dev/null +++ b/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(searchlib_searchlib_test_features + SOURCES + distance_closeness_fixture.cpp + DEPENDS + searchlib +) diff --git a/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp new file mode 100644 index 00000000000..76d40e14f48 --- /dev/null +++ b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "distance_closeness_fixture.h" + +namespace search::features::test { + +FeatureDumpFixture::~FeatureDumpFixture() = default; + +DistanceClosenessFixture::DistanceClosenessFixture(size_t fooCnt, size_t barCnt, const Labels &labels, const vespalib::string &featureName) + : queryEnv(&indexEnv), rankSetup(factory, indexEnv), + mdl(), match_data(), rankProgram(), fooHandles(), barHandles() +{ + for (size_t i = 0; i < fooCnt; ++i) { + uint32_t fieldId = indexEnv.getFieldByName("foo")->id(); + fooHandles.push_back(mdl.allocTermField(fieldId)); + SimpleTermData term; + term.setUniqueId(i + 1); + term.addField(fieldId).setHandle(fooHandles.back()); + queryEnv.getTerms().push_back(term); + } + for (size_t i = 0; i < barCnt; ++i) { + uint32_t fieldId = indexEnv.getFieldByName("bar")->id(); + barHandles.push_back(mdl.allocTermField(fieldId)); + SimpleTermData term; + term.setUniqueId(fooCnt + i + 1); + term.addField(fieldId).setHandle(barHandles.back()); + queryEnv.getTerms().push_back(term); + } + labels.inject(queryEnv.getProperties()); + rankSetup.setFirstPhaseRank(featureName); + rankSetup.setIgnoreDefaultRankFeatures(true); + ASSERT_TRUE(rankSetup.compile()); + match_data = mdl.createMatchData(); + rankProgram = rankSetup.create_first_phase_program(); + rankProgram->setup(*match_data, queryEnv); +} + +} + diff --git a/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.h b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.h new file mode 100644 index 00000000000..cdb1379659e --- /dev/null +++ b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.h @@ -0,0 +1,75 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/features/setup.h> +#include <vespa/searchlib/fef/fef.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> +#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> +#include <vespa/searchlib/fef/test/labels.h> +#include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/vespalib/testkit/test_kit.h> + +using namespace search::fef; +using namespace search::fef::test; + +using CollectionType = FieldInfo::CollectionType; +using DataType = FieldInfo::DataType; + +namespace search::features::test { + +struct BlueprintFactoryFixture { + BlueprintFactory factory; + BlueprintFactoryFixture() : factory() + { + setup_search_features(factory); + } +}; + +struct IndexEnvironmentFixture { + IndexEnvironment indexEnv; + IndexEnvironmentFixture() : indexEnv() + { + IndexEnvironmentBuilder builder(indexEnv); + builder.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "foo"); + builder.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::TENSOR, "bar"); + } +}; + +struct FeatureDumpFixture : public IDumpFeatureVisitor { + virtual void visitDumpFeature(const vespalib::string &) override { + TEST_ERROR("no features should be dumped"); + } + FeatureDumpFixture() : IDumpFeatureVisitor() {} + ~FeatureDumpFixture() override; +}; + +/** + * Fixture used by unit tests for distance and closeness rank features. + */ +struct DistanceClosenessFixture : BlueprintFactoryFixture, IndexEnvironmentFixture { + QueryEnvironment queryEnv; + RankSetup rankSetup; + MatchDataLayout mdl; + MatchData::UP match_data; + RankProgram::UP rankProgram; + std::vector<TermFieldHandle> fooHandles; + std::vector<TermFieldHandle> barHandles; + DistanceClosenessFixture(size_t fooCnt, size_t barCnt, const Labels &labels, const vespalib::string &featureName); + feature_t getScore(uint32_t docId) { + return Utils::getScoreFeature(*rankProgram, docId); + } + void setScore(TermFieldHandle handle, uint32_t docId, feature_t score) { + match_data->resolveTermField(handle)->setRawScore(docId, score); + } + void setFooScore(uint32_t i, uint32_t docId, feature_t distance) { + ASSERT_LESS(i, fooHandles.size()); + setScore(fooHandles[i], docId, 1.0/(1.0+distance)); + } + void setBarScore(uint32_t i, uint32_t docId, feature_t distance) { + ASSERT_LESS(i, barHandles.size()); + setScore(barHandles[i], docId, 1.0/(1.0+distance)); + } +}; + +} diff --git a/searchsummary/src/tests/docsumformat/docsum-pack.cpp b/searchsummary/src/tests/docsumformat/docsum-pack.cpp index ed1ba23017c..07aceea83e0 100644 --- a/searchsummary/src/tests/docsumformat/docsum-pack.cpp +++ b/searchsummary/src/tests/docsumformat/docsum-pack.cpp @@ -113,7 +113,7 @@ MyApp::Equal(GeneralResult *a, GeneralResult *b) void MyApp::TestIntValue(uint32_t line, GeneralResult *gres, const char *field, uint32_t value) { - ResEntry *entry = (gres != nullptr) ? gres->GetEntry(field) : nullptr; + ResEntry *entry = (gres != nullptr) ? gres->GetPresentEntry(field) : nullptr; bool rc = (entry != nullptr && entry->_type == RES_INT && @@ -125,7 +125,7 @@ MyApp::TestIntValue(uint32_t line, GeneralResult *gres, const char *field, uint3 void MyApp::TestDoubleValue(uint32_t line, GeneralResult *gres, const char *field, double value) { - ResEntry *entry = (gres != nullptr) ? gres->GetEntry(field) : nullptr; + ResEntry *entry = (gres != nullptr) ? gres->GetPresentEntry(field) : nullptr; bool rc = (entry != nullptr && entry->_type == RES_DOUBLE && @@ -137,7 +137,7 @@ MyApp::TestDoubleValue(uint32_t line, GeneralResult *gres, const char *field, do void MyApp::TestInt64Value(uint32_t line, GeneralResult *gres, const char *field, uint64_t value) { - ResEntry *entry = (gres != nullptr) ? gres->GetEntry(field) : nullptr; + ResEntry *entry = (gres != nullptr) ? gres->GetPresentEntry(field) : nullptr; bool rc = (entry != nullptr && entry->_type == RES_INT64 && @@ -150,7 +150,7 @@ MyApp::TestInt64Value(uint32_t line, GeneralResult *gres, const char *field, uin void MyApp::TestStringValue(uint32_t line, GeneralResult *gres, const char *field, const char *value) { - ResEntry *entry = (gres != nullptr) ? gres->GetEntry(field) : nullptr; + ResEntry *entry = (gres != nullptr) ? gres->GetPresentEntry(field) : nullptr; bool rc = (entry != nullptr && entry->_type == RES_STRING && @@ -168,7 +168,7 @@ MyApp::TestStringValue(uint32_t line, GeneralResult *gres, const char *field, co void MyApp::TestDataValue(uint32_t line, GeneralResult *gres, const char *field, const char *value) { - ResEntry *entry = (gres != nullptr) ? gres->GetEntry(field) : nullptr; + ResEntry *entry = (gres != nullptr) ? gres->GetPresentEntry(field) : nullptr; bool rc = (entry != nullptr && entry->_type == RES_DATA && diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp index d5a908fc8bd..7265dd89be4 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> -#include <vespa/searchsummary/docsummary/docsumfieldwriter.h> +#include <vespa/searchsummary/docsummary/docsum_field_writer.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/docsum_field_writer_state.h> #include <vespa/searchsummary/docsummary/attribute_combiner_dfw.h> @@ -24,7 +24,7 @@ using search::docsummary::AttributeCombinerDFW; using search::docsummary::GetDocsumsState; using search::docsummary::GetDocsumsStateCallback; using search::docsummary::IDocsumEnvironment; -using search::docsummary::IDocsumFieldWriter; +using search::docsummary::DocsumFieldWriter; using search::docsummary::test::MockAttributeManager; using search::docsummary::test::MockStateCallback; using search::docsummary::test::SlimeValue; @@ -34,7 +34,7 @@ namespace { struct AttributeCombinerTest : public ::testing::Test { MockAttributeManager attrs; - std::unique_ptr<IDocsumFieldWriter> writer; + std::unique_ptr<DocsumFieldWriter> writer; MockStateCallback callback; GetDocsumsState state; std::shared_ptr<search::MatchingElementsFields> _matching_elems_fields; diff --git a/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp index 67d505582d8..42443cf1058 100644 --- a/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp +++ b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp @@ -16,7 +16,7 @@ using search::attribute::BasicType; using search::attribute::CollectionType; using search::docsummary::AttributeDFWFactory; using search::docsummary::GetDocsumsState; -using search::docsummary::IDocsumFieldWriter; +using search::docsummary::DocsumFieldWriter; using search::docsummary::test::MockAttributeManager; using search::docsummary::test::MockStateCallback; using search::docsummary::test::SlimeValue; @@ -26,7 +26,7 @@ using ElementVector = std::vector<uint32_t>; class AttributeDFWTest : public ::testing::Test { protected: MockAttributeManager _attrs; - std::unique_ptr<IDocsumFieldWriter> _writer; + std::unique_ptr<DocsumFieldWriter> _writer; MockStateCallback _callback; GetDocsumsState _state; std::shared_ptr<search::MatchingElementsFields> _matching_elems_fields; diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index 675af283ee8..82aa9ceba92 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -16,8 +16,10 @@ #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchsummary/docsummary/docsum_store_document.h> +#include <vespa/searchsummary/docsummary/docsumstorevalue.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> +#include <vespa/searchsummary/docsummary/general_result.h> #include <vespa/searchsummary/docsummary/matched_elements_filter_dfw.h> #include <vespa/searchsummary/docsummary/resultconfig.h> #include <vespa/searchsummary/docsummary/resultpacker.h> @@ -215,7 +217,7 @@ public: { } ~MatchedElementsFilterTest(); - std::unique_ptr<IDocsumFieldWriter> make_field_writer(const std::string& input_field_name) { + std::unique_ptr<DocsumFieldWriter> make_field_writer(const std::string& input_field_name) { int input_field_enum = _doc_store.get_config().GetFieldNameEnum().Lookup(input_field_name.c_str()); return MatchedElementsFilterDFW::create(input_field_name, input_field_enum, _attr_ctx, _fields); diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index 11f0d0eb6d6..60584b26e31 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -4,7 +4,7 @@ #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchlib/common/matching_elements.h> -#include <vespa/searchsummary/docsummary/docsumfieldwriter.h> +#include <vespa/searchsummary/docsummary/docsum_field_writer.h> #include <vespa/searchsummary/docsummary/positionsdfw.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> #include <vespa/searchsummary/docsummary/docsumstate.h> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index 947fe9deeab..e3272fb36de 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -6,13 +6,15 @@ vespa_add_library(searchsummary_docsummary OBJECT attribute_field_writer.cpp attributedfw.cpp check_undefined_value_visitor.cpp + copy_dfw.cpp docsumconfig.cpp - docsumfieldwriter.cpp - docsumstate.cpp + docsum_field_writer.cpp docsum_store_document.cpp + docsumstate.cpp docsumstorevalue.cpp docsumwriter.cpp dynamicteaserdfw.cpp + empty_dfw.cpp general_result.cpp geoposdfw.cpp getdocsumargs.cpp @@ -22,10 +24,12 @@ vespa_add_library(searchsummary_docsummary OBJECT matched_elements_filter_dfw.cpp positionsdfw.cpp rankfeaturesdfw.cpp + res_type_utils.cpp resultclass.cpp resultconfig.cpp resultpacker.cpp searchdatatype.cpp + simple_dfw.cpp struct_fields_resolver.cpp struct_map_attribute_combiner_dfw.cpp summaryfeaturesdfw.cpp diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp index ff5c2c5e05b..f308795a1bc 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/util/stash.h> #include <algorithm> #include <cassert> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h index 18b4fd34e66..e5bed876b63 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h @@ -3,6 +3,7 @@ #pragma once #include "attribute_combiner_dfw.h" +#include <vector> namespace search::attribute { class IAttributeContext; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp index 79c11b20479..bf5578f38d6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp @@ -18,7 +18,7 @@ namespace search::docsummary { AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName, bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields) - : ISimpleDFW(), + : SimpleDFW(), _stateIndex(0), _filter_elements(filter_elements), _fieldName(fieldName), @@ -41,13 +41,13 @@ AttributeCombinerDFW::setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) return true; } -std::unique_ptr<IDocsumFieldWriter> +std::unique_ptr<DocsumFieldWriter> AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeContext &attrCtx, bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields) { StructFieldsResolver structFields(fieldName, attrCtx, true); if (structFields.has_error()) { - return std::unique_ptr<IDocsumFieldWriter>(); + return std::unique_ptr<DocsumFieldWriter>(); } else if (structFields.is_map_of_struct()) { return std::make_unique<StructMapAttributeCombinerDFW>(fieldName, structFields, filter_elements, std::move(matching_elems_fields)); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h index c1742595745..39f2d498c5b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h @@ -2,7 +2,8 @@ #pragma once -#include "docsumfieldwriter.h" +#include "simple_dfw.h" +#include <memory> namespace search { class MatchingElements; @@ -21,7 +22,7 @@ class DynamicDocsumWriter; * This class reads values from multiple struct field attributes and * inserts them as an array of struct or a map of struct. */ -class AttributeCombinerDFW : public ISimpleDFW +class AttributeCombinerDFW : public SimpleDFW { protected: uint32_t _stateIndex; @@ -36,8 +37,8 @@ public: ~AttributeCombinerDFW() override; bool IsGenerated() const override; bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) override; - static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, search::attribute::IAttributeContext &attrCtx, - bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields); + static std::unique_ptr<DocsumFieldWriter> create(const vespalib::string &fieldName, search::attribute::IAttributeContext &attrCtx, + bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields); void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index d5fdee096b1..e7b6acec646 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -2,6 +2,7 @@ #include "attributedfw.h" #include "docsumwriter.h" +#include "docsumstate.h" #include "docsum_field_writer_state.h" #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_codec.h> @@ -38,6 +39,24 @@ AttrDFW::AttrDFW(const vespalib::string & attrName) : { } +const attribute::IAttributeVector& +AttrDFW::get_attribute(const GetDocsumsState& s) const +{ + return *s.getAttribute(getIndex()); +} + +const vespalib::string & +AttrDFW::getAttributeName() const +{ + return _attrName; +} + +bool +AttrDFW::IsGenerated() const +{ + return true; +} + namespace { class SingleAttrDFW : public AttrDFW @@ -333,7 +352,7 @@ MultiAttrDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, vespa field_writer_state->insertField(docid, target); } -std::unique_ptr<IDocsumFieldWriter> +std::unique_ptr<DocsumFieldWriter> create_multi_writer(const IAttributeVector& attr, bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields) { auto type = attr.getBasicType(); @@ -355,7 +374,7 @@ create_multi_writer(const IAttributeVector& attr, bool filter_elements, std::sha } -std::unique_ptr<IDocsumFieldWriter> +std::unique_ptr<DocsumFieldWriter> AttributeDFWFactory::create(IAttributeManager& attr_mgr, const vespalib::string& attr_name, bool filter_elements, @@ -365,7 +384,7 @@ AttributeDFWFactory::create(IAttributeManager& attr_mgr, const auto* attr = ctx->getAttribute(attr_name); if (attr == nullptr) { Issue::report("No valid attribute vector found: '%s'", attr_name.c_str()); - return std::unique_ptr<IDocsumFieldWriter>(); + return std::unique_ptr<DocsumFieldWriter>(); } if (attr->hasMultiValue()) { return create_multi_writer(*attr, filter_elements, std::move(matching_elems_fields)); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h index 35f67fd5446..26351bdf501 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h @@ -2,38 +2,38 @@ #pragma once -#include "docsumfieldwriter.h" -#include "docsumstate.h" +#include "simple_dfw.h" +#include <memory> -namespace search { class MatchingElementsFields; } +namespace search { +class IAttributeManager; +class MatchingElementsFields; +} namespace search::attribute { class IAttributeVector; } namespace search::docsummary { /** - * Factory to create an IDocsumFieldWriter to write an attribute vector to slime. + * Factory to create an DocsumFieldWriter to write an attribute vector to slime. */ class AttributeDFWFactory { public: - static std::unique_ptr<IDocsumFieldWriter> create(IAttributeManager& attr_mgr, - const vespalib::string& attr_name, - bool filter_elements = false, - std::shared_ptr<MatchingElementsFields> matching_elems_fields - = std::shared_ptr<MatchingElementsFields>()); + static std::unique_ptr<DocsumFieldWriter> create(IAttributeManager& attr_mgr, + const vespalib::string& attr_name, + bool filter_elements = false, + std::shared_ptr<MatchingElementsFields> matching_elems_fields = std::shared_ptr<MatchingElementsFields>()); }; -class AttrDFW : public ISimpleDFW +class AttrDFW : public SimpleDFW { private: vespalib::string _attrName; protected: - const attribute::IAttributeVector& get_attribute(const GetDocsumsState& s) const { - return *s.getAttribute(getIndex()); - } - const vespalib::string & getAttributeName() const override { return _attrName; } + const attribute::IAttributeVector& get_attribute(const GetDocsumsState& s) const; + const vespalib::string & getAttributeName() const override; public: AttrDFW(const vespalib::string & attrName); - bool IsGenerated() const override { return true; } + bool IsGenerated() const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.cpp index e70b094aa64..836273ce3d8 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.cpp @@ -1,51 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "docsumfieldwriter.h" -#include "check_undefined_value_visitor.h" -#include "idocsumenvironment.h" -#include "docsumstate.h" -#include "summaryfieldconverter.h" -#include <vespa/searchlib/common/documentlocations.h> -#include <vespa/searchlib/common/location.h> -#include <vespa/searchlib/parsequery/stackdumpiterator.h> +#include "copy_dfw.h" +#include "general_result.h" +#include "i_docsum_store_document.h" +#include "resultconfig.h" #include <vespa/vespalib/data/slime/slime.h> #include <vespa/log/log.h> -LOG_SETUP(".searchlib.docsummary.docsumfieldwriter"); +LOG_SETUP(".searchlib.docsummary.copy_dfw"); namespace search::docsummary { -using search::attribute::IAttributeContext; -using search::attribute::IAttributeVector; -using search::attribute::BasicType; -using search::common::Location; - -//-------------------------------------------------------------------------- - -const vespalib::string IDocsumFieldWriter::_empty(""); - -bool -IDocsumFieldWriter::setFieldWriterStateIndex(uint32_t) -{ - return false; // Don't need any field writer state by default -} - -//-------------------------------------------------------------------------- - -EmptyDFW::EmptyDFW() = default; - -EmptyDFW::~EmptyDFW() = default; - -void -EmptyDFW::insertField(uint32_t, GetDocsumsState *, ResType, vespalib::slime::Inserter &target) -{ - // insert explicitly-empty field? - // target.insertNix(); - (void)target; -} - -//-------------------------------------------------------------------------- - CopyDFW::CopyDFW() : _inputFieldEnumValue(static_cast<uint32_t>(-1)), _input_field_name() @@ -64,10 +29,10 @@ CopyDFW::Init(const ResultConfig & config, const char *inputField) LOG(warning, "no docsum format contains field '%s'; copied fields will be empty", inputField); } - for (const auto & field : config) { - const ResConfigEntry *entry = field.GetEntry(field.GetIndexFromEnumValue(_inputFieldEnumValue)); + for (const auto & result_class : config) { + const ResConfigEntry *entry = result_class.GetEntry(result_class.GetIndexFromEnumValue(_inputFieldEnumValue)); - if (entry != nullptr && + if (entry != nullptr && !entry->_not_present && !IsRuntimeCompatible(entry->_type, RES_INT) && !IsRuntimeCompatible(entry->_type, RES_DOUBLE) && !IsRuntimeCompatible(entry->_type, RES_INT64) && @@ -75,7 +40,7 @@ CopyDFW::Init(const ResultConfig & config, const char *inputField) !IsRuntimeCompatible(entry->_type, RES_DATA)) { LOG(warning, "cannot use docsum field '%s' as input to copy; type conflict with result class %d (%s)", - inputField, field.GetClassID(), field.GetClassName()); + inputField, result_class.GetClassID(), result_class.GetClassName()); return false; } } @@ -87,19 +52,14 @@ CopyDFW::insertField(uint32_t /*docid*/, GeneralResult *gres, GetDocsumsState *, vespalib::slime::Inserter &target) { int idx = gres->GetClass()->GetIndexFromEnumValue(_inputFieldEnumValue); - ResEntry *entry = gres->GetEntry(idx); + ResEntry *entry = gres->GetPresentEntry(idx); if (entry == nullptr) { - auto input_field_value = gres->get_field_value(_input_field_name); - if (input_field_value) { - CheckUndefinedValueVisitor check_undefined; - input_field_value->accept(check_undefined); - if (!check_undefined.is_undefined()) { - SummaryFieldConverter::insert_summary_field(false, *input_field_value, target); - } + const auto* document = gres->get_document(); + if (document != nullptr) { + document->insert_summary_field(_input_field_name, target); } - } else if (IsRuntimeCompatible(entry->_type, type)) - { + } else if (IsRuntimeCompatible(entry->_type, type)) { switch (type) { case RES_INT: { uint32_t val32 = entry->_intval; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h new file mode 100644 index 00000000000..dab7417f60a --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "docsum_field_writer.h" + +namespace search::docsummary { + +class ResultConfig; + +/* + * Class for writing document summaries with content from another field. + */ +class CopyDFW : public DocsumFieldWriter +{ +private: + uint32_t _inputFieldEnumValue; + vespalib::string _input_field_name; + +public: + CopyDFW(); + ~CopyDFW() override; + + bool Init(const ResultConfig & config, const char *inputField); + + bool IsGenerated() const override { return false; } + void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, + vespalib::slime::Inserter &target) override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_blob_entry_filter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_blob_entry_filter.h new file mode 100644 index 00000000000..1d006386d35 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_blob_entry_filter.h @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "res_type.h" +#include <bitset> + +namespace search::docsummary { + +/* + * Class containing the set of result types not stored in docsum blobs. + * This is used for gradual migration towards elimination of docsum blobs. + */ +class DocsumBlobEntryFilter { + std::bitset<14> _skip_types; + +public: + DocsumBlobEntryFilter() + : _skip_types() + { + } + bool skip(ResType type) const noexcept { return _skip_types.test(type); } + DocsumBlobEntryFilter &add_skip(ResType type) { + _skip_types.set(type); + return *this; + } +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.cpp new file mode 100644 index 00000000000..c698f0603c6 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.cpp @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "docsum_field_writer.h" + +namespace search::docsummary { + +const vespalib::string DocsumFieldWriter::_empty(""); + +const vespalib::string& +DocsumFieldWriter::getAttributeName() const +{ + return _empty; +} + +bool +DocsumFieldWriter::isDefaultValue(uint32_t, const GetDocsumsState*) const +{ + return false; +} + +bool +DocsumFieldWriter::setFieldWriterStateIndex(uint32_t) +{ + return false; // Don't need any field writer state by default +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h new file mode 100644 index 00000000000..764f3507380 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "res_type_utils.h" +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib::slime { struct Inserter; } + +namespace search::docsummary { + +class GeneralResult; +class GetDocsumsState; + +/* + * Abstract class for writing document summaries. + */ +class DocsumFieldWriter +{ +public: + DocsumFieldWriter() + : _index(0) + { + } + virtual ~DocsumFieldWriter() = default; + static bool IsRuntimeCompatible(ResType a, ResType b) { + return ResTypeUtils::IsRuntimeCompatible(a, b); + } + virtual bool IsGenerated() const = 0; + virtual void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) = 0; + virtual const vespalib::string & getAttributeName() const; + virtual bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const; + void setIndex(size_t v) { _index = v; } + size_t getIndex() const { return _index; } + virtual bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex); +private: + size_t _index; + static const vespalib::string _empty; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp index c0b894ddc0a..e525989e972 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docsum_store_document.h" +#include "check_undefined_value_visitor.h" +#include "summaryfieldconverter.h" #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/document.h> @@ -28,4 +30,13 @@ DocsumStoreDocument::get_field_value(const vespalib::string& field_name) const return {}; } +void +DocsumStoreDocument::insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const +{ + auto field_value = get_field_value(field_name); + if (field_value) { + SummaryFieldConverter::insert_summary_field(*field_value, inserter); + } +} + } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h index 4508132e7e0..66a5a74fa8d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h @@ -18,6 +18,7 @@ public: DocsumStoreDocument(std::unique_ptr<document::Document> document); ~DocsumStoreDocument() override; std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const override; + void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index 24642c418fd..376d4f90204 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -1,8 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "attribute_combiner_dfw.h" #include "docsumconfig.h" +#include "attribute_combiner_dfw.h" +#include "copy_dfw.h" #include "docsumwriter.h" +#include "empty_dfw.h" #include "geoposdfw.h" #include "idocsumenvironment.h" #include "juniperdfw.h" @@ -10,6 +12,7 @@ #include "positionsdfw.h" #include "rankfeaturesdfw.h" #include "textextractordfw.h" +#include "summaryfeaturesdfw.h" #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> @@ -24,12 +27,12 @@ DynamicDocsumConfig::getResultConfig() const { return *_writer->GetResultConfig(); } -IDocsumFieldWriter::UP +std::unique_ptr<DocsumFieldWriter> DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<MatchingElementsFields> matching_elems_fields) { const ResultConfig & resultConfig = getResultConfig(); rc = false; - IDocsumFieldWriter::UP fieldWriter; + std::unique_ptr<DocsumFieldWriter> fieldWriter; if (overrideName == "dynamicteaser") { if ( ! argument.empty() ) { const char *langFieldName = "something unused"; @@ -127,7 +130,7 @@ DynamicDocsumConfig::configure(const vespa::config::search::SummarymapConfig &cf for (size_t i = 0; i < cfg.override.size(); ++i) { const vespa::config::search::SummarymapConfig::Override & o = cfg.override[i]; bool rc(false); - IDocsumFieldWriter::UP fieldWriter = createFieldWriter(o.field, o.command, o.arguments, rc, matching_elems_fields); + std::unique_ptr<DocsumFieldWriter> fieldWriter = createFieldWriter(o.field, o.command, o.arguments, rc, matching_elems_fields); if (rc && fieldWriter) { rc = _writer->Override(o.field.c_str(), fieldWriter.release()); // OBJECT HAND-OVER } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h index 70c8d524527..b86313dfbd4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h @@ -8,9 +8,9 @@ namespace search { class MatchingElementsFields; } namespace search::docsummary { class IDocsumEnvironment; +class DocsumFieldWriter; class DynamicDocsumWriter; class ResultConfig; -class IDocsumFieldWriter; class DynamicDocsumConfig { @@ -27,7 +27,7 @@ protected: const IDocsumEnvironment * getEnvironment() const { return _env; } const ResultConfig & getResultConfig() const; - virtual std::unique_ptr<IDocsumFieldWriter> + virtual std::unique_ptr<DocsumFieldWriter> createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<MatchingElementsFields> matching_elems_fields); private: diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h deleted file mode 100644 index bc135404de1..00000000000 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "general_result.h" -#include "resultconfig.h" -#include <vespa/searchlib/util/rawbuf.h> -#include <vespa/vespalib/data/slime/inserter.h> - -namespace search { class IAttributeManager; } - -namespace search::docsummary { - -class GetDocsumsState; - -class IDocsumFieldWriter -{ -public: - using UP = std::unique_ptr<IDocsumFieldWriter>; - IDocsumFieldWriter() : _index(0) { } - virtual ~IDocsumFieldWriter() = default; - - static bool IsRuntimeCompatible(ResType a, ResType b) { - return ResultConfig::IsRuntimeCompatible(a, b); - } - - virtual bool IsGenerated() const = 0; - virtual void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, - vespalib::slime::Inserter &target) = 0; - virtual const vespalib::string & getAttributeName() const { return _empty; } - virtual bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const { - (void) docid; - (void) state; - return false; - } - void setIndex(size_t v) { _index = v; } - size_t getIndex() const { return _index; } - virtual bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex); -private: - size_t _index; - static const vespalib::string _empty; -}; - -class ISimpleDFW : public IDocsumFieldWriter -{ -public: - virtual void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) = 0; - void insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state, ResType type, - vespalib::slime::Inserter &target) override - { - insertField(docid, state, type, target); - } -}; - -//-------------------------------------------------------------------------- - -class EmptyDFW : public ISimpleDFW -{ -public: - EmptyDFW(); - ~EmptyDFW() override; - - bool IsGenerated() const override { return true; } - void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; -}; - -//-------------------------------------------------------------------------- - -class CopyDFW : public IDocsumFieldWriter -{ -private: - uint32_t _inputFieldEnumValue; - vespalib::string _input_field_name; - -public: - CopyDFW(); - ~CopyDFW() override; - - bool Init(const ResultConfig & config, const char *inputField); - - bool IsGenerated() const override { return false; } - void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, - vespalib::slime::Inserter &target) override; -}; - -} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index de8bb36e98b..1492ce2b435 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docsumwriter.h" -#include "check_undefined_value_visitor.h" #include "docsumstate.h" #include "docsum_field_writer_state.h" +#include "i_docsum_store_document.h" #include "summaryfieldconverter.h" #include <vespa/document/fieldvalue/fieldvalue.h> #include <vespa/searchcommon/common/undefinedvalues.h> @@ -20,22 +20,6 @@ using vespalib::Issue; namespace search::docsummary { -namespace { - -void insert_document_field(const vespalib::string& field_name, const GeneralResult& gres, Inserter &inserter) -{ - auto input_field_value = gres.get_field_value(field_name); - if (input_field_value) { - CheckUndefinedValueVisitor check_undefined; - input_field_value->accept(check_undefined); - if (!check_undefined.is_undefined()) { - SummaryFieldConverter::insert_summary_field(false, *input_field_value, inserter); - } - } -} - -} - uint32_t IDocsumWriter::slime2RawBuf(const Slime & slime, RawBuf & buf) { @@ -116,7 +100,10 @@ static void convertEntry(const ResConfigEntry *resCfg, LOG_ASSERT(resCfg != nullptr); if (entry == nullptr || entry->_not_present) { // Entry is not present in docsum blob - insert_document_field(resCfg->_bindname, gres, inserter); + const auto* document = gres.get_document(); + if (document != nullptr) { + document->insert_summary_field(resCfg->_bindname, inserter); + } return; } @@ -181,7 +168,7 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, vespalib::slime::Cursor & docsum = topInserter.insertObject(); for (uint32_t i = 0; i < rci.outputClass->GetNumEntries(); ++i) { const ResConfigEntry *resCfg = rci.outputClass->GetEntry(i); - IDocsumFieldWriter *writer = _overrideTable[resCfg->_enumValue]; + DocsumFieldWriter *writer = _overrideTable[resCfg->_enumValue]; if (! writer->isDefaultValue(docid, state)) { const Memory field_name(resCfg->_bindname.data(), resCfg->_bindname.size()); ObjectInserter inserter(docsum, field_name); @@ -201,7 +188,7 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, vespalib::slime::Cursor & docsum = topInserter.insertObject(); for (uint32_t i = 0; i < rci.outputClass->GetNumEntries(); ++i) { const ResConfigEntry *outCfg = rci.outputClass->GetEntry(i); - IDocsumFieldWriter *writer = _overrideTable[outCfg->_enumValue]; + DocsumFieldWriter *writer = _overrideTable[outCfg->_enumValue]; const Memory field_name(outCfg->_bindname.data(), outCfg->_bindname.size()); ObjectInserter inserter(docsum, field_name); if (writer != nullptr) { @@ -214,13 +201,16 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, } else { int inIdx = rci.inputClass->GetIndexFromEnumValue(outCfg->_enumValue); const ResConfigEntry *inCfg = rci.inputClass->GetEntry(inIdx); - if (inCfg != nullptr && inCfg->_type == outCfg->_type) { + if (inCfg != nullptr && inCfg->_type == outCfg->_type && !inCfg->_not_present) { // copy field const ResEntry *entry = gres.GetEntry(inIdx); LOG_ASSERT(entry != nullptr); convertEntry(outCfg, entry, gres, inserter, slime); } else { - insert_document_field(outCfg->_bindname, gres, inserter); + const auto* document = gres.get_document(); + if (document != nullptr) { + document->insert_summary_field(outCfg->_bindname, inserter); + } } } } @@ -240,7 +230,7 @@ DynamicDocsumWriter::DynamicDocsumWriter( ResultConfig *config, KeywordExtractor { LOG_ASSERT(config != nullptr); _classInfoTable = new ResultClass::DynamicInfo[_numClasses]; - _overrideTable = new IDocsumFieldWriter*[_numEnumValues]; + _overrideTable = new DocsumFieldWriter*[_numEnumValues]; uint32_t i = 0; for (ResultConfig::iterator it(config->begin()), mt(config->end()); it != mt; it++, i++) { @@ -289,7 +279,7 @@ DynamicDocsumWriter::SetDefaultOutputClass(uint32_t classID) bool -DynamicDocsumWriter::Override(const char *fieldName, IDocsumFieldWriter *writer) +DynamicDocsumWriter::Override(const char *fieldName, DocsumFieldWriter *writer) { uint32_t fieldEnumValue = _resultConfig->GetFieldNameEnum().Lookup(fieldName); @@ -312,10 +302,10 @@ DynamicDocsumWriter::Override(const char *fieldName, IDocsumFieldWriter *writer) ++_numFieldWriterStates; } - for (auto & entry : *_resultConfig) { + for (auto & result_class : *_resultConfig) { - if (entry.GetIndexFromEnumValue(fieldEnumValue) >= 0) { - ResultClass::DynamicInfo *info = entry.getDynamicInfo(); + if (result_class.GetIndexFromEnumValue(fieldEnumValue) >= 0) { + ResultClass::DynamicInfo *info = result_class.getDynamicInfo(); info->_overrideCnt++; if (writer->IsGenerated()) info->_generateCnt++; @@ -334,7 +324,7 @@ DynamicDocsumWriter::InitState(IAttributeManager & attrMan, GetDocsumsState *sta state->_attributes.resize(_numEnumValues); state->_fieldWriterStates.resize(_numFieldWriterStates); for (size_t i(0); i < state->_attributes.size(); i++) { - const IDocsumFieldWriter *fw = _overrideTable[i]; + const DocsumFieldWriter *fw = _overrideTable[i]; if (fw) { const vespalib::string & attributeName = fw->getAttributeName(); if (!attributeName.empty()) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h index e70e3db8655..b3182221b68 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h @@ -7,12 +7,14 @@ #include "resultconfig.h" #include "docsumstore.h" #include "keywordextractor.h" -#include "docsumfieldwriter.h" +#include "docsum_field_writer.h" #include <vespa/searchlib/util/rawbuf.h> #include <vespa/fastlib/text/unicodeutil.h> #include <vespa/fastlib/text/wordfolder.h> -using search::IAttributeManager; +namespace search { class IAttributeManager; } + +namespace vespalib { class Slime; } namespace search::docsummary { @@ -36,7 +38,7 @@ public: }; virtual ~IDocsumWriter() {} - virtual void InitState(IAttributeManager & attrMan, GetDocsumsState *state) = 0; + virtual void InitState(search::IAttributeManager & attrMan, GetDocsumsState *state) = 0; virtual uint32_t WriteDocsum(uint32_t docid, GetDocsumsState *state, IDocsumStore *docinfos, search::RawBuf *target) = 0; virtual void insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState *state, @@ -58,7 +60,7 @@ private: uint32_t _numEnumValues; uint32_t _numFieldWriterStates; ResultClass::DynamicInfo *_classInfoTable; - IDocsumFieldWriter **_overrideTable; + DocsumFieldWriter** _overrideTable; void resolveInputClass(ResolveClassInfo &rci, uint32_t id) const; @@ -73,8 +75,8 @@ public: ResultConfig *GetResultConfig() { return _resultConfig; } bool SetDefaultOutputClass(uint32_t classID); - bool Override(const char *fieldName, IDocsumFieldWriter *writer); - void InitState(IAttributeManager & attrMan, GetDocsumsState *state) override; + bool Override(const char *fieldName, DocsumFieldWriter *writer); + void InitState(search::IAttributeManager & attrMan, GetDocsumsState *state) override; uint32_t WriteDocsum(uint32_t docid, GetDocsumsState *state, IDocsumStore *docinfos, search::RawBuf *target) override; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index ef1ffded941..57adbcc8163 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -332,7 +332,7 @@ JuniperTeaserDFW::Init( const ResConfigEntry *entry = it->GetEntry(it->GetIndexFromEnumValue(_inputFieldEnumValue)); - if (entry != nullptr && + if (entry != nullptr && !entry->_not_present && !IsRuntimeCompatible(entry->_type, RES_STRING) && !IsRuntimeCompatible(entry->_type, RES_DATA)) { @@ -347,7 +347,7 @@ JuniperTeaserDFW::Init( vespalib::stringref DynamicTeaserDFW::getJuniperInput(GeneralResult *gres) { int idx = gres->GetClass()->GetIndexFromEnumValue(_inputFieldEnumValue); - ResEntry *entry = gres->GetEntry(idx); + ResEntry *entry = gres->GetPresentEntry(idx); if (entry != nullptr) { const char *buf; uint32_t buflen; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.cpp new file mode 100644 index 00000000000..3d3b1e11626 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.cpp @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "empty_dfw.h" + +namespace search::docsummary { + +EmptyDFW::EmptyDFW() = default; + +EmptyDFW::~EmptyDFW() = default; + +void +EmptyDFW::insertField(uint32_t, GetDocsumsState *, ResType, vespalib::slime::Inserter &target) +{ + // insert explicitly-empty field? + // target.insertNix(); + (void)target; +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.h new file mode 100644 index 00000000000..d43eed8e9c6 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.h @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "simple_dfw.h" + +namespace search::docsummary { + +/* + * Class for writing empty document summaries. + */ +class EmptyDFW : public SimpleDFW +{ +public: + EmptyDFW(); + ~EmptyDFW() override; + + bool IsGenerated() const override { return true; } + void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp index 12391d26ce1..825c3b39c1b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp @@ -49,25 +49,17 @@ GeneralResult::~GeneralResult() } ResEntry * -GeneralResult::GetEntry(uint32_t idx) -{ - return (idx < _entrycnt) ? &_entries[idx] : nullptr; -} - -ResEntry * -GeneralResult::GetEntry(const char *name) +GeneralResult::GetPresentEntry(const char *name) { int idx = _resClass->GetIndexFromName(name); - - return (idx >= 0 && (uint32_t)idx < _entrycnt) ? &_entries[idx] : nullptr; + return GetPresentEntry(idx); } - ResEntry * -GeneralResult::GetEntryFromEnumValue(uint32_t value) +GeneralResult::GetPresentEntryFromEnumValue(uint32_t value) { int idx = _resClass->GetIndexFromEnumValue(value); - return (idx >= 0 && (uint32_t)idx < _entrycnt) ? &_entries[idx] : nullptr; + return GetPresentEntry(idx); } std::unique_ptr<document::FieldValue> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.h b/searchsummary/src/vespa/searchsummary/docsummary/general_result.h index 93114dbb44d..cff27a496e3 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.h @@ -32,9 +32,16 @@ public: ~GeneralResult(); const ResultClass *GetClass() const { return _resClass; } - ResEntry *GetEntry(uint32_t idx); - ResEntry *GetEntry(const char *name); - ResEntry *GetEntryFromEnumValue(uint32_t val); + ResEntry *GetEntry(uint32_t idx) { return (idx < _entrycnt) ? &_entries[idx] : nullptr; } + ResEntry *GetPresentEntry(uint32_t idx) { + if (idx >= _entrycnt) { + return nullptr; + } + ResEntry* entry = &_entries[idx]; + return entry->_not_present ? nullptr : entry; + } + ResEntry *GetPresentEntry(const char *name); + ResEntry *GetPresentEntryFromEnumValue(uint32_t val); std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const; bool unpack(const char *buf, const size_t buflen); @@ -46,6 +53,8 @@ public: return false; } } + + const IDocsumStoreDocument *get_document() const noexcept { return _document; } }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp index 8f627ac1b9a..c3806d6e7ea 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/common/location.h> #include <vespa/vespalib/util/jsonwriter.h> #include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/issue.h> #include <climits> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h index 3fbf54b18a9..f81412eb34c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h @@ -7,6 +7,8 @@ namespace document { class FieldValue; } +namespace vespalib::slime { struct Inserter; } + namespace search::docsummary { /** @@ -19,6 +21,7 @@ class IDocsumStoreDocument public: virtual ~IDocsumStoreDocument() = default; virtual std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const = 0; + virtual void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const = 0; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h index d9a657038c4..5e2ec517a47 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h @@ -4,14 +4,14 @@ #include "general_result.h" #include "resultconfig.h" -#include "docsumfieldwriter.h" +#include "docsum_field_writer.h" #include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/juniper/rpinterface.h> namespace search::docsummary { -class JuniperDFW : public IDocsumFieldWriter +class JuniperDFW : public DocsumFieldWriter { public: virtual bool Init( @@ -21,7 +21,7 @@ public: const char *inputField); protected: JuniperDFW(juniper::Juniper * juniper); - virtual ~JuniperDFW(); + ~JuniperDFW() override; uint32_t _inputFieldEnumValue; std::unique_ptr<juniper::Config> _juniperConfig; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp index fb53ddcc470..c05fec7a0ce 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp @@ -1,7 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "docsumstate.h" #include "matched_elements_filter_dfw.h" +#include "docsumstate.h" +#include "general_result.h" #include "struct_fields_resolver.h" #include "summaryfieldconverter.h" #include <vespa/document/fieldvalue/document.h> @@ -40,21 +41,21 @@ MatchedElementsFilterDFW::MatchedElementsFilterDFW(const std::string& input_fiel { } -std::unique_ptr<IDocsumFieldWriter> +std::unique_ptr<DocsumFieldWriter> MatchedElementsFilterDFW::create(const std::string& input_field_name, uint32_t input_field_enum, std::shared_ptr<MatchingElementsFields> matching_elems_fields) { return std::make_unique<MatchedElementsFilterDFW>(input_field_name, input_field_enum, std::move(matching_elems_fields)); } -std::unique_ptr<IDocsumFieldWriter> +std::unique_ptr<DocsumFieldWriter> MatchedElementsFilterDFW::create(const std::string& input_field_name, uint32_t input_field_enum, search::attribute::IAttributeContext& attr_ctx, std::shared_ptr<MatchingElementsFields> matching_elems_fields) { StructFieldsResolver resolver(input_field_name, attr_ctx, false); if (resolver.has_error()) { - return std::unique_ptr<IDocsumFieldWriter>(); + return std::unique_ptr<DocsumFieldWriter>(); } resolver.apply_to(*matching_elems_fields); return std::make_unique<MatchedElementsFilterDFW>(input_field_name, input_field_enum, std::move(matching_elems_fields)); @@ -93,7 +94,7 @@ filter_matching_elements_in_input_field_while_converting_to_slime(const FieldVal bool resolve_input_field_as_slime(GeneralResult& result, int entry_idx, Slime& input_field_as_slime) { - ResEntry* entry = result.GetEntry(entry_idx); + ResEntry* entry = result.GetPresentEntry(entry_idx); if (entry != nullptr) { decode_input_field_to_slime(*entry, input_field_as_slime); return true; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h index 505a2557408..b117da541d6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h @@ -2,7 +2,11 @@ #pragma once -#include "docsumfieldwriter.h" +#include "docsum_field_writer.h" +#include <memory> +#include <vector> + +namespace search { class MatchingElementsFields; } namespace search::attribute { class IAttributeContext; } @@ -13,7 +17,7 @@ namespace search::docsummary { * (array of primitive, weighted set of primitive, map of primitives, map of struct, array of struct) * that is retrieved from the document store. */ -class MatchedElementsFilterDFW : public IDocsumFieldWriter { +class MatchedElementsFilterDFW : public DocsumFieldWriter { private: std::string _input_field_name; uint32_t _input_field_enum; @@ -24,11 +28,11 @@ private: public: MatchedElementsFilterDFW(const std::string& input_field_name, uint32_t input_field_enum, std::shared_ptr<MatchingElementsFields> matching_elems_fields); - static std::unique_ptr<IDocsumFieldWriter> create(const std::string& input_field_name, uint32_t input_field_enum, - std::shared_ptr<MatchingElementsFields> matching_elems_fields); - static std::unique_ptr<IDocsumFieldWriter> create(const std::string& input_field_name, uint32_t input_field_enum, - search::attribute::IAttributeContext& attr_ctx, - std::shared_ptr<MatchingElementsFields> matching_elems_fields); + static std::unique_ptr<DocsumFieldWriter> create(const std::string& input_field_name, uint32_t input_field_enum, + std::shared_ptr<MatchingElementsFields> matching_elems_fields); + static std::unique_ptr<DocsumFieldWriter> create(const std::string& input_field_name, uint32_t input_field_enum, + search::attribute::IAttributeContext& attr_ctx, + std::shared_ptr<MatchingElementsFields> matching_elems_fields); ~MatchedElementsFilterDFW(); bool IsGenerated() const override { return false; } void insertField(uint32_t docid, GeneralResult* result, GetDocsumsState *state, diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index 1fcb0a49be1..7f3a929a62f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -270,8 +270,9 @@ PositionsDFW::UP PositionsDFW::create(const char *attribute_name, IAttributeMana return std::make_unique<PositionsDFW>(attribute_name, useV8geoPositions); } -AbsDistanceDFW::UP AbsDistanceDFW::create(const char *attribute_name, IAttributeManager *attribute_manager) { - AbsDistanceDFW::UP ret; +std::unique_ptr<DocsumFieldWriter> +AbsDistanceDFW::create(const char *attribute_name, IAttributeManager *attribute_manager) { + std::unique_ptr<DocsumFieldWriter> ret; if (attribute_manager != nullptr) { if (!attribute_name) { LOG(debug, "createAbsDistanceDFW: missing attribute name '%p'", attribute_name); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h index b3e041c1379..d9445abd2ff 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h @@ -45,7 +45,7 @@ public: void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; - static UP create(const char *attribute_name, IAttributeManager *index_man); + static std::unique_ptr<DocsumFieldWriter> create(const char *attribute_name, IAttributeManager *index_man); }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp index 38b58ef94fc..5d3b104189b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp @@ -3,6 +3,7 @@ #include "rankfeaturesdfw.h" #include "docsumstate.h" #include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/data/slime/inserter.h> namespace search::docsummary { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h index eab9fab60b2..91f9e80d303 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h @@ -2,11 +2,13 @@ #pragma once -#include "summaryfeaturesdfw.h" +#include "simple_dfw.h" namespace search::docsummary { -class RankFeaturesDFW : public ISimpleDFW +class IDocsumEnvironment; + +class RankFeaturesDFW : public SimpleDFW { private: IDocsumEnvironment * _env; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_type.h b/searchsummary/src/vespa/searchsummary/docsummary/res_type.h new file mode 100644 index 00000000000..02c9f1522a4 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_type.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search::docsummary { + +/** + * This enumeration contains values denoting the different types of + * docsum fields. NOTE: The internal implementation depends on RES_INT + * having the value 0. All types < RES_STRING must be fixed size and + * all types > RES_STRING must be variable size. + **/ +enum ResType { + RES_INT = 0, + RES_SHORT, + RES_BOOL, + RES_BYTE, + RES_FLOAT, + RES_DOUBLE, + RES_INT64, + RES_STRING, + RES_DATA, + RES_LONG_STRING, + RES_LONG_DATA, + RES_JSONSTRING, + RES_TENSOR, + RES_FEATUREDATA +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.cpp b/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.cpp new file mode 100644 index 00000000000..98cc8372ac1 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.cpp @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "res_type_utils.h" + +namespace search::docsummary { + +const char * +ResTypeUtils::GetResTypeName(ResType type) +{ + switch (type) { + case RES_INT: return "integer"; + case RES_SHORT: return "short"; + case RES_BYTE: return "byte"; + case RES_BOOL: return "bool"; + case RES_FLOAT: return "float"; + case RES_DOUBLE: return "double"; + case RES_INT64: return "int64"; + case RES_STRING: return "string"; + case RES_DATA: return "data"; + case RES_LONG_STRING: return "longstring"; + case RES_LONG_DATA: return "longdata"; + case RES_JSONSTRING: return "jsonstring"; + case RES_TENSOR: return "tensor"; + case RES_FEATUREDATA: return "featuredata"; + } + return "unknown-type"; +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.h b/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.h new file mode 100644 index 00000000000..194a008c179 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.h @@ -0,0 +1,102 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "res_type.h" + +namespace search::docsummary { + + +/* + * Utilitiy functions for checking if result type is ok. + */ +struct ResTypeUtils +{ + /** + * Determine if a result field type is of variable size. + * + * @return true for variable size field types, false for fixed + * size field types + **/ + static bool IsVariableSize(ResType t) { return (t >= RES_STRING); } + + + /** + * Determine if a pair of result field types are binary + * compatible. A pair of types are binary compatible if the packed + * representation is identical. + * + * @return true if the given types are binary compatible. + * @param a enum value of a result field type. + * @param b enum value of a result field type. + **/ + static bool IsBinaryCompatible(ResType a, ResType b) + { + if (a == b) { + return true; + } + switch (a) { + case RES_BYTE: + case RES_BOOL: + return (b == RES_BYTE || b == RES_BOOL); + case RES_STRING: + case RES_DATA: + return (b == RES_STRING || b == RES_DATA); + case RES_LONG_STRING: + case RES_LONG_DATA: + case RES_FEATUREDATA: + case RES_JSONSTRING: + return (b == RES_LONG_STRING || b == RES_LONG_DATA || + b == RES_FEATUREDATA || b == RES_JSONSTRING); + default: + return false; + } + return false; + } + + + /** + * Determine if a pair of result field types are runtime + * compatible. A pair of types are runtime compatible if the + * unpacked (@ref ResEntry) representation is identical. + * + * @return true if the given types are runtime compatible. + * @param a enum value of a result field type. + * @param b enum value of a result field type. + **/ + static bool IsRuntimeCompatible(ResType a, ResType b) + { + switch (a) { + case RES_INT: + case RES_SHORT: + case RES_BYTE: + case RES_BOOL: + return (b == RES_INT || b == RES_SHORT || b == RES_BYTE || b == RES_BOOL); + case RES_FLOAT: + case RES_DOUBLE: + return (b == RES_FLOAT || b == RES_DOUBLE); + case RES_INT64: + return b == RES_INT64; + case RES_STRING: + case RES_LONG_STRING: + case RES_JSONSTRING: + return (b == RES_STRING || b == RES_LONG_STRING || b == RES_JSONSTRING); + case RES_DATA: + case RES_LONG_DATA: + return (b == RES_DATA || b == RES_LONG_DATA); + case RES_TENSOR: + return (b == RES_TENSOR); + case RES_FEATUREDATA: + return (b == RES_FEATUREDATA); + } + return false; + } + + /** + * @return the name of the given result field type. + * @param resType enum value of a result field type. + **/ + static const char *GetResTypeName(ResType type); +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp index ed1cd8b542b..65db72efc40 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp @@ -26,11 +26,7 @@ int ResultClass::GetIndexFromName(const char* name) const { NameIdMap::const_iterator found(_nameMap.find(name)); - if (found == _nameMap.end()) { - return -1; - } - int idx = found->second; - return _entries[idx]._not_present ? -1 : idx; + return (found != _nameMap.end()) ? found->second : -1; } bool diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h index 8865e28acc6..47feed70e97 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h @@ -2,57 +2,15 @@ #pragma once +#include "docsum_blob_entry_filter.h" #include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/searchlib/util/stringenum.h> -#include <bitset> namespace search::docsummary { /** - * This enumeration contains values denoting the different types of - * docsum fields. NOTE: The internal implementation depends on RES_INT - * having the value 0. All types < RES_STRING must be fixed size and - * all types > RES_STRING must be variable size. - **/ -enum ResType { - RES_INT = 0, - RES_SHORT, - RES_BOOL, - RES_BYTE, - RES_FLOAT, - RES_DOUBLE, - RES_INT64, - RES_STRING, - RES_DATA, - RES_LONG_STRING, - RES_LONG_DATA, - RES_JSONSTRING, - RES_TENSOR, - RES_FEATUREDATA -}; - -/* - * Class containing the set of result types not stored in docsum blobs. - * This is used for gradual migration towards elimination of docsum blobs. - */ -class DocsumBlobEntryFilter { - std::bitset<14> _skip_types; - -public: - DocsumBlobEntryFilter() - : _skip_types() - { - } - bool skip(ResType type) const noexcept { return _skip_types.test(type); } - DocsumBlobEntryFilter &add_skip(ResType type) { - _skip_types.set(type); - return *this; - } -}; - -/** * This struct describes a single docsum field (name and type). A * docsum blob is unpacked into an array of ResEntry instances * by interpreting it as described by an array of ResConfigEntry @@ -238,7 +196,7 @@ public: * GeneralResult::GetEntry(string) method; no need to call it * directly. * - * @return field index or -1 if not found or _not_present is set. + * @return field index or -1 if not found **/ int GetIndexFromName(const char* name) const; @@ -255,15 +213,11 @@ public: * call it directly. NOTE3: You need to call the CreateEnumMap * method before calling this one. * - * @return field index or -1 if not found or _not_present is set. + * @return field index or -1 if not found **/ int GetIndexFromEnumValue(uint32_t value) const { - if (value >= _enumMap.size()) { - return -1; - } - int idx = _enumMap[value]; - return ((idx < 0) || _entries[idx]._not_present) ? -1 : idx; + return (value < _enumMap.size()) ? _enumMap[value] : -1; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index 4096e26a6e3..168b4c81374 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "resultconfig.h" +#include "resultclass.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <atomic> @@ -45,28 +46,6 @@ ResultConfig::~ResultConfig() } -const char * -ResultConfig::GetResTypeName(ResType type) -{ - switch (type) { - case RES_INT: return "integer"; - case RES_SHORT: return "short"; - case RES_BYTE: return "byte"; - case RES_BOOL: return "bool"; - case RES_FLOAT: return "float"; - case RES_DOUBLE: return "double"; - case RES_INT64: return "int64"; - case RES_STRING: return "string"; - case RES_DATA: return "data"; - case RES_LONG_STRING: return "longstring"; - case RES_LONG_DATA: return "longdata"; - case RES_JSONSTRING: return "jsonstring"; - case RES_TENSOR: return "tensor"; - case RES_FEATUREDATA: return "featuredata"; - } - return "unknown-type"; -} - void ResultConfig::Reset() { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h index 1438aee73ce..945eef8514f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h @@ -2,14 +2,15 @@ #pragma once -#include "resultclass.h" -#include "general_result.h" +#include "docsum_blob_entry_filter.h" +#include "res_type_utils.h" #include <vespa/config-summary.h> -#include <vespa/searchlib/util/rawbuf.h> #include <vespa/searchlib/util/stringenum.h> namespace search::docsummary { +class ResultClass; + /** * This class represents the overall result configuration. A result * configuration may contain multiple result classes, where each @@ -31,7 +32,7 @@ private: ResultConfig& operator=(const ResultConfig &); typedef vespalib::hash_map<vespalib::string, uint32_t> NameMap; - typedef vespalib::hash_map<uint32_t, ResultClass::UP> IdMap; + typedef vespalib::hash_map<uint32_t, std::unique_ptr<ResultClass>> IdMap; uint32_t _defaultSummaryId; bool _useV8geoPositions; search::util::StringEnum _fieldEnum; @@ -95,85 +96,9 @@ public: static uint32_t NoClassID() { return static_cast<uint32_t>(-1); } - /** - * Determine if a result field type is of variable size. - * - * @return true for variable size field types, false for fixed - * size field types - **/ - static bool IsVariableSize(ResType t) { return (t >= RES_STRING); } - - - /** - * Determine if a pair of result field types are binary - * compatible. A pair of types are binary compatible if the packed - * representation is identical. - * - * @return true if the given types are binary compatible. - * @param a enum value of a result field type. - * @param b enum value of a result field type. - **/ - static bool IsBinaryCompatible(ResType a, ResType b) - { - if (a == b) { - return true; - } - switch (a) { - case RES_BYTE: - case RES_BOOL: - return (b == RES_BYTE || b == RES_BOOL); - case RES_STRING: - case RES_DATA: - return (b == RES_STRING || b == RES_DATA); - case RES_LONG_STRING: - case RES_LONG_DATA: - case RES_FEATUREDATA: - case RES_JSONSTRING: - return (b == RES_LONG_STRING || b == RES_LONG_DATA || - b == RES_FEATUREDATA || b == RES_JSONSTRING); - default: - return false; - } - return false; - } - - - /** - * Determine if a pair of result field types are runtime - * compatible. A pair of types are runtime compatible if the - * unpacked (@ref ResEntry) representation is identical. - * - * @return true if the given types are runtime compatible. - * @param a enum value of a result field type. - * @param b enum value of a result field type. - **/ - static bool IsRuntimeCompatible(ResType a, ResType b) - { - switch (a) { - case RES_INT: - case RES_SHORT: - case RES_BYTE: - case RES_BOOL: - return (b == RES_INT || b == RES_SHORT || b == RES_BYTE || b == RES_BOOL); - case RES_FLOAT: - case RES_DOUBLE: - return (b == RES_FLOAT || b == RES_DOUBLE); - case RES_INT64: - return b == RES_INT64; - case RES_STRING: - case RES_LONG_STRING: - case RES_JSONSTRING: - return (b == RES_STRING || b == RES_LONG_STRING || b == RES_JSONSTRING); - case RES_DATA: - case RES_LONG_DATA: - return (b == RES_DATA || b == RES_LONG_DATA); - case RES_TENSOR: - return (b == RES_TENSOR); - case RES_FEATUREDATA: - return (b == RES_FEATUREDATA); - } - return false; - } + static bool IsVariableSize(ResType t) { return ResTypeUtils::IsVariableSize(t); } + static bool IsBinaryCompatible(ResType a, ResType b) { return ResTypeUtils::IsBinaryCompatible(a, b); } + static bool IsRuntimeCompatible(ResType a, ResType b) { return ResTypeUtils::IsRuntimeCompatible(a, b); } // whether last config seen wanted useV8geoPositions = true static bool wantedV8geoPositions(); @@ -182,7 +107,7 @@ public: * @return the name of the given result field type. * @param resType enum value of a result field type. **/ - static const char *GetResTypeName(ResType type); + static const char *GetResTypeName(ResType type) { return ResTypeUtils::GetResTypeName(type); } /** * Discard the current configuration and start over. After this diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp index 4cf36785f69..66b15c1c8a9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "resultpacker.h" +#include "resultconfig.h" #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h index f2460f3d3c3..816433652b8 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h @@ -2,9 +2,14 @@ #pragma once -#include "resultconfig.h" +#include "res_type_utils.h" +#include "resultclass.h" +#include <vespa/searchlib/util/rawbuf.h> namespace search::docsummary { + +class ResultConfig; + /** * An Object of this class may be used to create docsum blobs. A * single blob is created by first indicating what result class the @@ -27,11 +32,8 @@ private: const ResConfigEntry *_cfgEntry; // current field of current blob bool _error; // error flag for current blob - static const char *GetResTypeName(ResType type) - { return ResultConfig::GetResTypeName(type); } - - static bool IsBinaryCompatible(ResType a, ResType b) - { return ResultConfig::IsBinaryCompatible(a, b); } + static const char *GetResTypeName(ResType type) { return ResTypeUtils::GetResTypeName(type); } + static bool IsBinaryCompatible(ResType a, ResType b) { return ResTypeUtils::IsBinaryCompatible(a, b); } void WarnType(ResType type) const; void SetFormatError(ResType type); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp new file mode 100644 index 00000000000..8cc872378bd --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "simple_dfw.h" + +namespace search::docsummary { + +void +SimpleDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) +{ + insertField(docid, state, type, target); +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h new file mode 100644 index 00000000000..abebb7de2b8 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "docsum_field_writer.h" + +namespace search::docsummary { + +/* + * Abstract class for writing document summaries that don't need + * access to a document retrieved from IDocsumStore. + */ +class SimpleDFW : public DocsumFieldWriter +{ +public: + virtual void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) = 0; + void insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp index aec55977546..38a9cc8c50b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/util/stash.h> #include <algorithm> #include <cassert> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index f920b7be0cd..c28e986612c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -3,6 +3,7 @@ #include "summaryfeaturesdfw.h" #include "docsumstate.h" #include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.summaryfeaturesdfw"); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h index 6c4084b0221..d12feb69182 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h @@ -2,13 +2,13 @@ #pragma once -#include "docsumfieldwriter.h" +#include "simple_dfw.h" namespace search::docsummary { class IDocsumEnvironment; -class SummaryFeaturesDFW : public ISimpleDFW +class SummaryFeaturesDFW : public SimpleDFW { private: IDocsumEnvironment * _env; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp index 6082c82e863..1fcd0213ce2 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "summaryfieldconverter.h" +#include "check_undefined_value_visitor.h" #include "linguisticsannotation.h" #include "resultconfig.h" #include "searchdatatype.h" @@ -594,10 +595,14 @@ SummaryFieldConverter::convert_field_with_filter(bool markup, } void -SummaryFieldConverter::insert_summary_field(bool markup, const FieldValue& value, vespalib::slime::Inserter& inserter) +SummaryFieldConverter::insert_summary_field(const FieldValue& value, vespalib::slime::Inserter& inserter) { - SlimeFiller visitor(inserter, markup); - value.accept(visitor); + CheckUndefinedValueVisitor check_undefined; + value.accept(check_undefined); + if (!check_undefined.is_undefined()) { + SlimeFiller visitor(inserter, false); + value.accept(visitor); + } } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h index 4367dbcd109..23d20b23c1f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h @@ -25,7 +25,7 @@ public: const document::FieldValue& value, const std::vector<uint32_t>& matching_elems); - static void insert_summary_field(bool markup, const document::FieldValue& value, vespalib::slime::Inserter& inserter); + static void insert_summary_field(const document::FieldValue& value, vespalib::slime::Inserter& inserter); }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp index c85a4fb5788..dc6d9524ee4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp @@ -1,8 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "textextractordfw.h" -#include "tokenizer.h" #include "docsumstate.h" +#include "general_result.h" +#include "tokenizer.h" +#include "resultconfig.h" +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.textextractordfw"); @@ -31,7 +34,7 @@ TextExtractorDFW::insertField(uint32_t, GeneralResult *gres, GetDocsumsState *, vespalib::slime::Inserter &target) { vespalib::string extracted; - ResEntry * entry = gres->GetEntryFromEnumValue(_inputFieldEnum); + ResEntry * entry = gres->GetPresentEntryFromEnumValue(_inputFieldEnum); if (entry != nullptr) { const char * buf = nullptr; uint32_t buflen = 0; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h index 10764e5c21d..3bce2ae5cd7 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h @@ -2,14 +2,16 @@ #pragma once -#include "docsumfieldwriter.h" +#include "docsum_field_writer.h" namespace search::docsummary { +class ResultConfig; + /** * This is the docsum field writer used to extract the original text from a disk summary on the juniper format. **/ -class TextExtractorDFW : public IDocsumFieldWriter +class TextExtractorDFW : public DocsumFieldWriter { private: TextExtractorDFW(const TextExtractorDFW &); @@ -19,7 +21,7 @@ private: public: TextExtractorDFW(); - ~TextExtractorDFW() {} + ~TextExtractorDFW() override = default; bool init(const vespalib::string & fieldName, const vespalib::string & inputField, const ResultConfig & config); bool IsGenerated() const override { return false; } void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp index 7e0b96b1d82..fcb56c4a553 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp @@ -106,11 +106,11 @@ PersistenceProviderWrapper::putAsync(const spi::Bucket& bucket, spi::Timestamp t } void -PersistenceProviderWrapper::removeAsync(const spi::Bucket& bucket, std::vector<TimeStampAndDocumentId> ids, +PersistenceProviderWrapper::removeAsync(const spi::Bucket& bucket, std::vector<spi::IdAndTimestamp> ids, spi::OperationComplete::UP onComplete) { - for (const TimeStampAndDocumentId & stampedId : ids) { - LOG_SPI("remove(" << bucket << ", " << stampedId.first << ", " << stampedId.second << ")"); + for (const auto & stampedId : ids) { + LOG_SPI("remove(" << bucket << ", " << stampedId.timestamp << ", " << stampedId.id << ")"); } CHECK_ERROR_ASYNC(spi::RemoveResult, FAIL_REMOVE, onComplete); _spi.removeAsync(bucket, std::move(ids), std::move(onComplete)); diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.h b/storage/src/tests/persistence/common/persistenceproviderwrapper.h index 3c93bc91d85..ec7ca70d7c7 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.h +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.h @@ -106,7 +106,7 @@ public: spi::BucketIdListResult listBuckets(BucketSpace bucketSpace) const override; spi::BucketInfoResult getBucketInfo(const spi::Bucket&) const override; void putAsync(const spi::Bucket&, spi::Timestamp, spi::DocumentSP, spi::OperationComplete::UP) override; - void removeAsync(const spi::Bucket&, std::vector<TimeStampAndDocumentId> ids, spi::OperationComplete::UP) override; + void removeAsync(const spi::Bucket&, std::vector<spi::IdAndTimestamp> ids, spi::OperationComplete::UP) override; void removeIfFoundAsync(const spi::Bucket&, spi::Timestamp, const spi::DocumentId&, spi::OperationComplete::UP) override; void updateAsync(const spi::Bucket&, spi::Timestamp, spi::DocumentUpdateSP, spi::OperationComplete::UP) override; spi::GetResult get(const spi::Bucket&, const document::FieldSet&, const spi::DocumentId&, spi::Context&) const override; diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def index 75a3344b618..3083fb37081 100644 --- a/storage/src/vespa/storage/config/stor-communicationmanager.def +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -38,20 +38,17 @@ mbus.tcp_no_delay bool default=true restart ## Number of threads for network. mbus.num_network_threads int default=1 restart -## Number of workers threads for messagebus -## Any value below 1 will be 1. -mbus.num_threads int default=4 restart - ## The number of events in the queue of a network (FNET) thread before it is woken up. mbus.events_before_wakeup int default=1 restart ## Enable to use above thread pool for encoding replies ## False will use network(fnet) thread +## Deprecated and void mbus.dispatch_on_encode bool default=true restart ## Enable to use above thread pool for decoding replies ## False will use network(fnet) thread -## Todo: Change default once verified in large scale deployment. +## Deprecated and void mbus.dispatch_on_decode bool default=true restart ## The number of network (FNET) threads used by the shared rpc resource. diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index f5d29fb32a7..d5bf733a30c 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -114,13 +114,13 @@ bucketStatesAreSemanticallyEqual(const api::BucketInfo& a, const api::BucketInfo class UnrevertableRemoveEntryProcessor : public BucketProcessor::EntryProcessor { public: - using DocumentIdsAndTimeStamps = std::vector<std::pair<spi::Timestamp, spi::DocumentId>>; + using DocumentIdsAndTimeStamps = std::vector<spi::IdAndTimestamp>; UnrevertableRemoveEntryProcessor(DocumentIdsAndTimeStamps & to_remove) : _to_remove(to_remove) {} void process(spi::DocEntry& entry) override { - _to_remove.emplace_back(entry.getTimestamp(), *entry.getDocumentId()); + _to_remove.emplace_back(*entry.getDocumentId(), entry.getTimestamp()); } private: DocumentIdsAndTimeStamps & _to_remove; diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 012d5c2619d..ae68a694c90 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -522,9 +522,9 @@ MergeHandler::applyDiffEntry(std::shared_ptr<ApplyBucketDiffState> async_results _clock, _env._metrics.merge_handler_metrics.put_latency); _spi.putAsync(bucket, timestamp, std::move(doc), std::move(complete)); } else { - std::vector<spi::PersistenceProvider::TimeStampAndDocumentId> ids; - ids.emplace_back(timestamp, e._docName); - auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), ids[0].second, + std::vector<spi::IdAndTimestamp> ids; + ids.emplace_back(document::DocumentId(e._docName), timestamp); + auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), ids[0].id, std::move(throttle_token), "remove", _clock, _env._metrics.merge_handler_metrics.remove_latency); _spi.removeAsync(bucket, std::move(ids), std::move(complete)); diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index 1be9679c641..9e55c0f9088 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -152,7 +152,7 @@ ProviderErrorWrapper::putAsync(const spi::Bucket &bucket, spi::Timestamp ts, spi } void -ProviderErrorWrapper::removeAsync(const spi::Bucket &bucket, std::vector<TimeStampAndDocumentId> ids, +ProviderErrorWrapper::removeAsync(const spi::Bucket &bucket, std::vector<spi::IdAndTimestamp> ids, spi::OperationComplete::UP onComplete) { onComplete->addResultHandler(this); diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.h b/storage/src/vespa/storage/persistence/provider_error_wrapper.h index 7bd406a8758..82447fe4549 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.h +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.h @@ -58,7 +58,7 @@ public: void register_error_listener(std::shared_ptr<ProviderErrorListener> listener); void putAsync(const spi::Bucket &, spi::Timestamp, spi::DocumentSP, spi::OperationComplete::UP) override; - void removeAsync(const spi::Bucket&, std::vector<TimeStampAndDocumentId>, spi::OperationComplete::UP) override; + void removeAsync(const spi::Bucket&, std::vector<spi::IdAndTimestamp>, spi::OperationComplete::UP) override; void removeIfFoundAsync(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::OperationComplete::UP) override; void updateAsync(const spi::Bucket &, spi::Timestamp, spi::DocumentUpdateSP, spi::OperationComplete::UP) override; void setActiveStateAsync(const spi::Bucket& b, spi::BucketInfo::ActiveState newState, spi::OperationComplete::UP onComplete) override; diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index a88c339052d..7fb6685a8b5 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -352,12 +352,9 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> LOG(debug, "setting up slobrok config from id: '%s", _configUri.getConfigId().c_str()); mbus::RPCNetworkParams params(_configUri); params.setConnectionExpireSecs(config->mbus.rpctargetcache.ttl); - params.setNumThreads(std::max(1, config->mbus.numThreads)); params.setNumNetworkThreads(std::max(1, config->mbus.numNetworkThreads)); params.setNumRpcTargets(std::max(1, config->mbus.numRpcTargets)); params.events_before_wakeup(std::max(1, config->mbus.eventsBeforeWakeup)); - params.setDispatchOnDecode(config->mbus.dispatchOnDecode); - params.setDispatchOnEncode(config->mbus.dispatchOnEncode); params.setTcpNoDelay(config->mbus.tcpNoDelay); params.setIdentity(mbus::Identity(_component.getIdentity())); diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp index ce0706a03fb..fac801550d4 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp @@ -1,14 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vsm/vsm/docsumconfig.h> -#include <vespa/searchsummary/docsummary/docsumfieldwriter.h> +#include <vespa/searchsummary/docsummary/copy_dfw.h> +#include <vespa/searchsummary/docsummary/empty_dfw.h> #include <vespa/searchsummary/docsummary/matched_elements_filter_dfw.h> +#include <vespa/searchsummary/docsummary/resultconfig.h> #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vsm/config/config-vsmfields.h> #include <vespa/vsm/config/config-vsmsummary.h> using search::MatchingElementsFields; -using search::docsummary::IDocsumFieldWriter; +using search::docsummary::DocsumFieldWriter; using search::docsummary::CopyDFW; using search::docsummary::EmptyDFW; using search::docsummary::MatchedElementsFilterDFW; @@ -41,10 +43,10 @@ DynamicDocsumConfig::DynamicDocsumConfig(search::docsummary::IDocsumEnvironment* { } -IDocsumFieldWriter::UP +std::unique_ptr<DocsumFieldWriter> DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<search::MatchingElementsFields> matching_elems_fields) { - IDocsumFieldWriter::UP fieldWriter; + std::unique_ptr<DocsumFieldWriter> fieldWriter; if ((overrideName == "staticrank") || (overrideName == "ranklog") || (overrideName == "label") || diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h index 11010c04e90..a660c544d7d 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h @@ -20,7 +20,7 @@ private: public: DynamicDocsumConfig(search::docsummary::IDocsumEnvironment* env, search::docsummary::DynamicDocsumWriter* writer, std::shared_ptr<VsmfieldsConfig> vsm_fields_config); private: - std::unique_ptr<search::docsummary::IDocsumFieldWriter> + std::unique_ptr<search::docsummary::DocsumFieldWriter> createFieldWriter(const string & fieldName, const string & overrideName, const string & cf, bool & rc, std::shared_ptr<search::MatchingElementsFields> matching_elems_fields) override; }; diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index a5fe08da605..20083b9160e 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -134,6 +134,7 @@ public: DocsumStoreVsmDocument(const document::Document* document); ~DocsumStoreVsmDocument() override; std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const override; + void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const override; }; DocsumStoreVsmDocument::DocsumStoreVsmDocument(const document::Document* document) @@ -158,6 +159,15 @@ DocsumStoreVsmDocument::get_field_value(const vespalib::string& field_name) cons return {}; } +void +DocsumStoreVsmDocument::insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const +{ + auto field_value = get_field_value(field_name); + if (field_value) { + SummaryFieldConverter::insert_summary_field(*field_value, inserter); + } +} + } FieldPath diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp index 14f44527887..add52effac2 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -138,6 +138,12 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot) // init result config DocsumBlobEntryFilter docsum_blob_entry_filter; docsum_blob_entry_filter.add_skip(search::docsummary::RES_INT); + docsum_blob_entry_filter.add_skip(search::docsummary::RES_SHORT); + docsum_blob_entry_filter.add_skip(search::docsummary::RES_BYTE); + docsum_blob_entry_filter.add_skip(search::docsummary::RES_FLOAT); + docsum_blob_entry_filter.add_skip(search::docsummary::RES_DOUBLE); + docsum_blob_entry_filter.add_skip(search::docsummary::RES_INT64); + docsum_blob_entry_filter.add_skip(search::docsummary::RES_TENSOR); std::unique_ptr<ResultConfig> resCfg(new ResultConfig(docsum_blob_entry_filter)); if ( ! resCfg->ReadConfig(*summary.get(), _configId.c_str())) { throw std::runtime_error("(re-)configuration of VSM (docsum tools) failed due to bad summary config"); diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java index 084d0e9185d..915f8dd67d5 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java @@ -108,6 +108,7 @@ public class DistributionTestFactory extends CrossPlatformTestFactory { } } + @SuppressWarnings("deprecation") private static StorDistributionConfig.Builder deserializeConfig(String s) { return new StorDistributionConfig.Builder( new ConfigGetter<>(StorDistributionConfig.class).getConfig("raw:" + s)); diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 8060a2d67a1..4dd0ade7499 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -220,19 +220,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - <executions> - <execution> - <id>attach-sources</id> - <goals> - <goal>jar</goal> - </goals> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> <release>${vespaClients.jdk.releaseVersion}</release> diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml index ebb1240a198..0032fe33de0 100644 --- a/vespa-osgi-testrunner/pom.xml +++ b/vespa-osgi-testrunner/pom.xml @@ -80,6 +80,11 @@ <artifactId>org.apache.felix.framework</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <scope>provided</scope> + </dependency> </dependencies> <build> diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java index 31dbd939991..ef4b402d33b 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.testrunner; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.component.annotation.Inject; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.EmptyResponse; @@ -9,10 +11,6 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.exception.ExceptionUtils; import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.testrunner.TestReport.ContainerNode; import com.yahoo.vespa.testrunner.TestReport.FailureNode; import com.yahoo.vespa.testrunner.TestReport.NamedNode; import com.yahoo.vespa.testrunner.TestReport.Node; @@ -22,6 +20,7 @@ import com.yahoo.yolean.Exceptions; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintStream; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -43,6 +42,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; public class TestRunnerHandler extends ThreadedHttpRequestHandler { private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS"); + private static final JsonFactory factory = new JsonFactory(); private final TestRunner testRunner; @@ -80,15 +80,13 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { long fetchRecordsAfter = Optional.ofNullable(request.getProperty("after")) .map(Long::parseLong) .orElse(-1L); - return new SlimeJsonResponse(logToSlime(testRunner.getLog(fetchRecordsAfter))); + return new CustomJsonResponse(out -> render(out, testRunner.getLog(fetchRecordsAfter))); case "/tester/v1/status": return new MessageResponse(testRunner.getStatus().name()); case "/tester/v1/report": TestReport report = testRunner.getReport(); - if (report == null) - return new EmptyResponse(200); - - return new SlimeJsonResponse(toSlime(report)); + if (report == null) return new EmptyResponse(204); + else return new CustomJsonResponse(out -> render(out, report)); } return new MessageResponse(Status.NOT_FOUND, "Not found: " + request.getUri().getPath()); } @@ -114,31 +112,30 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { return path.substring(lastSlash + 1); } - static Slime logToSlime(Collection<LogRecord> log) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor recordArray = root.setArray("logRecords"); - logArrayToSlime(recordArray, log); - return slime; - } - - static void logArrayToSlime(Cursor recordArray, Collection<LogRecord> log) { - log.forEach(record -> { - Cursor recordObject = recordArray.addObject(); - recordObject.setLong("id", record.getSequenceNumber()); - recordObject.setLong("at", record.getMillis()); - recordObject.setString("type", typeOf(record.getLevel())); + private static void render(OutputStream out, Collection<LogRecord> log) throws IOException { + var json = factory.createGenerator(out); + json.writeStartObject(); + json.writeArrayFieldStart("logRecords"); + for (LogRecord record : log) { String message = record.getMessage() == null ? "" : record.getMessage(); if (record.getThrown() != null) { ByteArrayOutputStream buffer = new ByteArrayOutputStream(); record.getThrown().printStackTrace(new PrintStream(buffer)); message += (message.isEmpty() ? "" : "\n") + buffer; } - recordObject.setString("message", message); - }); + json.writeStartObject(); + json.writeNumberField("id", record.getSequenceNumber()); + json.writeNumberField("at", record.getMillis()); + json.writeStringField("type", typeOf(record.getLevel())); + json.writeStringField("message", message); + json.writeEndObject(); + } + json.writeEndArray(); + json.writeEndObject(); + json.close(); } - public static String typeOf(Level level) { + private static String typeOf(Level level) { return level.getName().equals("html") ? "html" : level.intValue() < Level.INFO.intValue() ? "debug" : level.intValue() < Level.WARNING.intValue() ? "info" @@ -146,72 +143,105 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { : "error"; } - private static Slime toSlime(TestReport report) { - var slime = new Slime(); - var root = slime.setObject(); + private static void render(OutputStream out, TestReport report) throws IOException { + JsonGenerator json = factory.createGenerator(out); + json.writeStartObject(); + + json.writeFieldName("report"); + render(json, (Node) report.root()); + + // TODO jonmv: remove + json.writeObjectFieldStart("summary"); + + renderSummary(json, report); - toSlime(root.setObject("report"), (Node) report.root()); + json.writeArrayFieldStart("failures"); + renderFailures(json, report.root()); + json.writeEndArray(); + + json.writeEndObject(); // TODO jonmv: remove - Map<TestReport.Status, Long> tally = report.root().tally(); - var summary = root.setObject("summary"); - summary.setLong("success", tally.getOrDefault(TestReport.Status.successful, 0L)); - summary.setLong("failed", tally.getOrDefault(TestReport.Status.failed, 0L) + tally.getOrDefault(TestReport.Status.error, 0L)); - summary.setLong("ignored", tally.getOrDefault(TestReport.Status.skipped, 0L)); - summary.setLong("aborted", tally.getOrDefault(TestReport.Status.aborted, 0L)); - summary.setLong("inconclusive", tally.getOrDefault(TestReport.Status.inconclusive, 0L)); - toSlime(summary.setArray("failures"), root.setArray("output"), report.root()); + json.writeArrayFieldStart("output"); + renderOutput(json, report.root()); + json.writeEndArray(); - return slime; + json.writeEndObject(); + json.close(); } - static void toSlime(Cursor failuresArray, Cursor outputArray, Node node) { - for (Node child : node.children()) - TestRunnerHandler.toSlime(failuresArray, outputArray, child); + private static void renderSummary(JsonGenerator json, TestReport report) throws IOException { + Map<TestReport.Status, Long> tally = report.root().tally(); + json.writeNumberField("success", tally.getOrDefault(TestReport.Status.successful, 0L)); + json.writeNumberField("failed", tally.getOrDefault(TestReport.Status.failed, 0L) + tally.getOrDefault(TestReport.Status.error, 0L)); + json.writeNumberField("ignored", tally.getOrDefault(TestReport.Status.skipped, 0L)); + json.writeNumberField("aborted", tally.getOrDefault(TestReport.Status.aborted, 0L)); + json.writeNumberField("inconclusive", tally.getOrDefault(TestReport.Status.inconclusive, 0L)); + } + private static void renderFailures(JsonGenerator json, Node node) throws IOException { if (node instanceof FailureNode) { - Cursor failureObject = failuresArray.addObject(); - failureObject.setString("testName", node.parent.name()); - failureObject.setString("testError", ((FailureNode) node).thrown().getMessage()); - failureObject.setString("exception", ExceptionUtils.getStackTraceAsString(((FailureNode) node).thrown())); + json.writeStartObject(); + json.writeStringField("testName", node.parent.name()); + json.writeStringField("testError", ((FailureNode) node).thrown().getMessage()); + json.writeStringField("exception", ExceptionUtils.getStackTraceAsString(((FailureNode) node).thrown())); + json.writeEndObject(); } - if (node instanceof OutputNode) + else { + for (Node child : node.children()) + renderFailures(json, child); + } + } + + private static void renderOutput(JsonGenerator json, Node node) throws IOException { + if (node instanceof OutputNode) { for (LogRecord record : ((OutputNode) node).log()) if (record.getMessage() != null) - outputArray.addString(formatter.format(record.getInstant().atOffset(ZoneOffset.UTC)) + " " + record.getMessage()); + json.writeString(formatter.format(record.getInstant().atOffset(ZoneOffset.UTC)) + " " + record.getMessage()); + } + else { + for (Node child : node.children()) + renderOutput(json, child); + } } - static void toSlime(Cursor nodeObject, Node node) { - if (node instanceof NamedNode) toSlime(nodeObject, (NamedNode) node); - if (node instanceof OutputNode) toSlime(nodeObject, (OutputNode) node); + private static void render(JsonGenerator json, Node node) throws IOException { + json.writeStartObject(); + if (node instanceof NamedNode) render(json, (NamedNode) node); + if (node instanceof OutputNode) render(json, (OutputNode) node); if ( ! node.children().isEmpty()) { - Cursor childrenArray = nodeObject.setArray("children"); - for (Node child : node.children) - toSlime(childrenArray.addObject(), child); + json.writeArrayFieldStart("children"); + for (Node child : node.children) { + render(json, child); + } + json.writeEndArray(); } + json.writeEndObject(); } - static void toSlime(Cursor nodeObject, NamedNode node) { + private static void render(JsonGenerator json, NamedNode node) throws IOException { String type = node instanceof FailureNode ? "failure" : node instanceof TestNode ? "test" : "container"; - nodeObject.setString("type", type); - nodeObject.setString("name", node.name()); - nodeObject.setString("status", node.status().name()); - nodeObject.setLong("start", node.start().toEpochMilli()); - nodeObject.setLong("duration", node.duration().toMillis()); + json.writeStringField("type", type); + json.writeStringField("name", node.name()); + json.writeStringField("status", node.status().name()); + json.writeNumberField("start", node.start().toEpochMilli()); + json.writeNumberField("duration", node.duration().toMillis()); } - static void toSlime(Cursor nodeObject, OutputNode node) { - nodeObject.setString("type", "output"); - Cursor childrenArray = nodeObject.setArray("children"); + private static void render(JsonGenerator json, OutputNode node) throws IOException { + json.writeStringField("type", "output"); + json.writeArrayFieldStart("children"); for (LogRecord record : node.log()) { - Cursor recordObject = childrenArray.addObject(); - recordObject.setString("message", (record.getLoggerName() == null ? "" : record.getLoggerName() + ": ") + - (record.getMessage() != null ? record.getMessage() : "") + - (record.getThrown() != null ? (record.getMessage() != null ? "\n" : "") + traceToString(record.getThrown()) : "")); - recordObject.setLong("at", record.getInstant().toEpochMilli()); - recordObject.setString("level", typeOf(record.getLevel())); + json.writeStartObject(); + json.writeStringField("message", (record.getLoggerName() == null ? "" : record.getLoggerName() + ": ") + + (record.getMessage() != null ? record.getMessage() : "") + + (record.getThrown() != null ? (record.getMessage() != null ? "\n" : "") + traceToString(record.getThrown()) : "")); + json.writeNumberField("at", record.getInstant().toEpochMilli()); + json.writeStringField("level", typeOf(record.getLevel())); + json.writeEndObject(); } + json.writeEndArray(); } private static String traceToString(Throwable thrown) { @@ -220,4 +250,36 @@ public class TestRunnerHandler extends ThreadedHttpRequestHandler { return buffer.toString(UTF_8); } + private interface Renderer { + + void render(OutputStream out) throws IOException; + + } + + private static class CustomJsonResponse extends HttpResponse { + + private final Renderer renderer; + + private CustomJsonResponse(Renderer renderer) { + super(200); + this.renderer = renderer; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + renderer.render(outputStream); + } + + @Override + public String getContentType() { + return "application/json"; + } + + @Override + public long maxPendingBytes() { + return 1 << 25; // 32MB + } + + } + } diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java index bc878353d4b..b0e5119c06e 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/SampleTest.java @@ -71,7 +71,7 @@ public class SampleTest { @Test void successful() { log.log(new Level("html", INFO.intValue()) { }, "<body />"); - log.log(INFO, "Very informative"); + log.log(INFO, "Very informative: \"\\n\": \n"); log.log(WARNING, "Oh no", new IllegalArgumentException("error", new RuntimeException("wrapped"))); } diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java index 5ce737d7649..6d6fbbf2cf1 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java @@ -27,7 +27,6 @@ import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; -import java.util.logging.Level; import java.util.logging.LogRecord; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; @@ -136,14 +135,6 @@ class TestRunnerHandlerTest { } } - /* Creates a LogRecord that has a known instant and sequence number to get predictable serialization results. */ - private static LogRecord logRecord(String logMessage) { - LogRecord logRecord = new LogRecord(Level.INFO, logMessage); - logRecord.setInstant(testInstant); - logRecord.setSequenceNumber(0); - return logRecord; - } - private static class MockRunner implements TestRunner { private final TestRunner.Status status; diff --git a/vespa-osgi-testrunner/src/test/resources/output.json b/vespa-osgi-testrunner/src/test/resources/output.json index 847ae5800e9..2b4aa9e5599 100644 --- a/vespa-osgi-testrunner/src/test/resources/output.json +++ b/vespa-osgi-testrunner/src/test/resources/output.json @@ -1,133 +1,133 @@ { "logRecords": [ { - "id": 18, + "id": 2, "at": 0, "type": "info", "message": "spam" }, { - "id": 21, + "id": 5, "at": 0, "type": "info", "message": "spam" }, { - "id": 22, + "id": 6, "at": 0, "type": "error", "message": "java.lang.NoClassDefFoundError\n\tat com.yahoo.vespa.test.samples.SampleTest.error(SampleTest.java:87)\n" }, { - "id": 25, + "id": 9, "at": 0, "type": "info", "message": "spam" }, { - "id": 26, + "id": 10, "at": 0, "type": "info", "message": "I have a bad feeling about this" }, { - "id": 27, + "id": 11, "at": 0, "type": "error", "message": "org.opentest4j.AssertionFailedError: baz ==> expected: <foo> but was: <bar>\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:55)\n\tat org.junit.jupiter.api.AssertionUtils.failNotEqual(AssertionUtils.java:62)\n\tat org.junit.jupiter.api.AssertEquals.assertEquals(AssertEquals.java:182)\n\tat org.junit.jupiter.api.Assertions.assertEquals(Assertions.java:1152)\n\tat com.yahoo.vespa.test.samples.SampleTest.failing(SampleTest.java:81)\n" }, { - "id": 31, + "id": 15, "at": 0, "type": "info", "message": "spam" }, { - "id": 32, + "id": 16, "at": 0, "type": "info", "message": "I'm here with Erwin today; Erwin, what can you tell us about your cat?" }, { - "id": 33, + "id": 17, "at": 0, "type": "warning", "message": "ai.vespa.hosted.cd.InconclusiveTestException: the cat is both dead _and_ alive\n\tat com.yahoo.vespa.test.samples.SampleTest.inconclusive(SampleTest.java:93)\n" }, { - "id": 36, + "id": 20, "at": 0, "type": "info", "message": "spam" }, { - "id": 37, + "id": 21, "at": 0, "type": "info", "message": "<body />" }, { - "id": 38, + "id": 22, "at": 0, "type": "info", - "message": "Very informative" + "message": "Very informative: \"\\n\": \n" }, { - "id": 39, + "id": 23, "at": 0, "type": "warning", "message": "Oh no\njava.lang.IllegalArgumentException: error\n\tat com.yahoo.vespa.test.samples.SampleTest.successful(SampleTest.java:75)\nCaused by: java.lang.RuntimeException: wrapped\n\t... 1 more\n" }, { - "id": 43, + "id": 27, "at": 0, "type": "info", "message": "spam" }, { - "id": 46, + "id": 30, "at": 0, "type": "info", "message": "Catch me if you can!" }, { - "id": 50, + "id": 34, "at": 0, "type": "error", "message": "org.opentest4j.AssertionFailedError: no charm\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:39)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:134)\n\tat com.yahoo.vespa.test.samples.SampleTest$Inner.lambda$others$1(SampleTest.java:105)\n" }, { - "id": 54, + "id": 38, "at": 0, "type": "info", "message": "spam" }, { - "id": 2, + "id": 67, "at": 0, "type": "error", "message": "org.opentest4j.AssertionFailedError\n\tat org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:35)\n\tat org.junit.jupiter.api.Assertions.fail(Assertions.java:115)\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.test(FailingTestAndBothAftersTest.java:19)\n\tSuppressed: java.lang.RuntimeException\n\t\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.moreFail(FailingTestAndBothAftersTest.java:16)\n" }, { - "id": 4, + "id": 69, "at": 0, "type": "error", "message": "java.lang.RuntimeException\n\tat com.yahoo.vespa.test.samples.FailingTestAndBothAftersTest.fail(FailingTestAndBothAftersTest.java:13)\n" }, { - "id": 7, + "id": 72, "at": 0, "type": "error", "message": "org.junit.platform.commons.JUnitException: @BeforeAll method 'void com.yahoo.vespa.test.samples.WrongBeforeAllTest.wrong()' must be static unless the test class is annotated with @TestInstance(Lifecycle.PER_CLASS).\n" }, { - "id": 11, + "id": 76, "at": 0, "type": "error", "message": "java.lang.NullPointerException\n\tat com.yahoo.vespa.test.samples.FailingExtensionTest$FailingExtension.<init>(FailingExtensionTest.java:19)\n" }, { - "id": 15, + "id": 80, "at": 12000, "type": "error", "message": "java.lang.ClassNotFoundException: School's out all summer!\n" diff --git a/vespa-osgi-testrunner/src/test/resources/report.json b/vespa-osgi-testrunner/src/test/resources/report.json index fa76c222f93..443694e2e0c 100644 --- a/vespa-osgi-testrunner/src/test/resources/report.json +++ b/vespa-osgi-testrunner/src/test/resources/report.json @@ -191,7 +191,7 @@ "level": "info" }, { - "message": "com.yahoo.vespa.test.samples.SampleTest: Very informative", + "message": "com.yahoo.vespa.test.samples.SampleTest: Very informative: \"\\n\": \n", "at": 0, "level": "info" }, @@ -546,7 +546,7 @@ "00:00:00.000 I'm here with Erwin today; Erwin, what can you tell us about your cat?", "00:00:00.000 spam", "00:00:00.000 <body />", - "00:00:00.000 Very informative", + "00:00:00.000 Very informative: \"\\n\": \n", "00:00:00.000 Oh no", "00:00:00.000 spam", "00:00:00.000 Catch me if you can!", diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java index f3ea8fb5a80..2fc34112517 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java @@ -25,7 +25,7 @@ public class FeedHandlerTest { FeedHandler handler = new FeedHandler( new RejectingContainerThreadpool(), new CollectingMetric(), - new DocumentTypeManager(new DocumentmanagerConfig.Builder().enablecompression(true).build()), + new DocumentTypeManager(new DocumentmanagerConfig.Builder().build()), null /* session cache */, MetricReceiver.nullImplementation); var responseHandler = new RequestHandlerTestDriver.MockResponseHandler(); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java index dcabc1f338e..5b8b5b1827f 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java @@ -108,7 +108,7 @@ public class FeedHandlerV3Test { } private FeedHandlerV3 setupFeederHandler(Executor threadPool) { - DocumentTypeManager docMan = new DocumentTypeManager(new DocumentmanagerConfig.Builder().enablecompression(true).build()); + DocumentTypeManager docMan = new DocumentTypeManager(new DocumentmanagerConfig.Builder().build()); FeedHandlerV3 feedHandlerV3 = new FeedHandlerV3( threadPool, metric, diff --git a/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterList.java b/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterList.java index d60463df581..c42d48b4821 100644 --- a/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterList.java +++ b/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterList.java @@ -20,6 +20,7 @@ public class ClusterList { this.contentClusters = List.copyOf(contentClusters); } + @SuppressWarnings("deprecation") public ClusterList(String configId) { this(new ConfigGetter<>(ClusterListConfig.class).getConfig(configId)); } |
