diff options
author | Erlend <erlendniko@hotmail.com> | 2022-08-10 14:17:34 +0200 |
---|---|---|
committer | Erlend <erlendniko@hotmail.com> | 2022-08-10 14:17:34 +0200 |
commit | b2b99295d1ee48ba29f902c28cef468eea57b144 (patch) | |
tree | c5bdb05fad9713274e8f72c119338e666f317ab0 | |
parent | 0d0ad9b655496480bcbb4f7566cc0b1b0a66f85d (diff) | |
parent | b0fdd9f2e8885ed21cbf584d6e3b1218ba6ae4d2 (diff) |
Merge remote-tracking branch 'upstream/master'
59 files changed, 1014 insertions, 1442 deletions
diff --git a/client/js/app/package.json b/client/js/app/package.json index 030fe818ba6..f88c4234ad5 100644 --- a/client/js/app/package.json +++ b/client/js/app/package.json @@ -32,9 +32,11 @@ "eslint-plugin-react-perf": "^3", "eslint-plugin-unused-imports": "^2", "husky": "^7", + "lodash": "^4", "prettier": "2", "pretty-quick": "^3", - "react-router-dom": "^6.3.0", + "react-router-dom": "^6", + "use-context-selector": "^1", "vite": "^2" } } diff --git a/client/js/app/src/app/main.jsx b/client/js/app/src/app/main.jsx index 08f86115d40..96514d419e1 100644 --- a/client/js/app/src/app/main.jsx +++ b/client/js/app/src/app/main.jsx @@ -1,12 +1,9 @@ import React from 'react'; import ReactDOM from 'react-dom/client'; import { App } from 'app/app'; -import { ResponseProvider } from './pages/querybuilder/Components/Contexts/ResponseContext'; ReactDOM.createRoot(document.getElementById('root')).render( <React.StrictMode> - <ResponseProvider> - <App /> - </ResponseProvider> + <App /> </React.StrictMode> ); diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/AddPropertyButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/AddPropertyButton.jsx deleted file mode 100644 index 47b5a67875b..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Buttons/AddPropertyButton.jsx +++ /dev/null @@ -1,46 +0,0 @@ -import React, { useContext, useState } from 'react'; -import { QueryInputContext } from '../Contexts/QueryInputContext'; -import SimpleButton from './SimpleButton'; - -export default function AddPropertyButton({ id }) { - const { inputs, setInputs, childMap } = useContext(QueryInputContext); - const [childId, setChildId] = useState(1); - - /** - * Add a child to the input that has the provided id - */ - const addChildProperty = () => { - const newInputs = inputs.slice(); - let currentId = id.substring(0, 1); - let index = newInputs.findIndex((element) => element.id === currentId); //get the index of the root parent - let children = newInputs[index].children; - let parentType = newInputs[index].type; - for (let i = 3; i < id.length + 1; i += 2) { - currentId = id.substring(0, i); - index = children.findIndex((element) => element.id === currentId); - parentType = parentType + '_' + children[index].type; - children = children[index].children; - } - let type = childMap[parentType]; - children.push({ - id: id + '.' + childId, - type: type[Object.keys(type)[0]].name, - typeof: type[Object.keys(type)[0]].type, - input: '', - hasChildren: false, - children: [], - }); - setInputs(newInputs); - setChildId((childId) => childId + 1); - }; - - return ( - <SimpleButton - id={`propb${id}`} - className={'addpropsbutton'} - onClick={addChildProperty} - > - + Add property - </SimpleButton> - ); -} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/AddQueryInputButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/AddQueryInputButton.jsx deleted file mode 100644 index 8aeaff971bf..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Buttons/AddQueryInputButton.jsx +++ /dev/null @@ -1,48 +0,0 @@ -import React, { useContext } from 'react'; -import { QueryInputContext } from '../Contexts/QueryInputContext'; -import OverlayTrigger from 'react-bootstrap/OverlayTrigger'; -import Tooltip from 'react-bootstrap/Tooltip'; - -export default function AddQueryInput() { - const { inputs, setInputs, id, setId } = useContext(QueryInputContext); - - /** - * Adds a new element to inputs. - * @param {Event} e Event that happened. - */ - const updateInputs = (e) => { - e.preventDefault(); - setId((id) => id + 1); - setInputs((prevInputs) => [ - ...prevInputs, - { - id: `${id + 1}`, - type: 'yql', - typeof: 'String', - input: '', - hasChildren: false, - children: [], - }, - ]); - }; - - return ( - <OverlayTrigger - placement="right" - delay={{ show: 250, hide: 400 }} - overlay={<Tooltip id="button-tooltip">Add row</Tooltip>} - > - <span> - <button - id="addRow" - className="addRow" - height="0" - width="0" - onClick={updateInputs} - > - + - </button> - </span> - </OverlayTrigger> - ); -} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/CopyResponseButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/CopyResponseButton.jsx index d01daa7b0d6..2c53c4dd1e6 100644 --- a/client/js/app/src/app/pages/querybuilder/Components/Buttons/CopyResponseButton.jsx +++ b/client/js/app/src/app/pages/querybuilder/Components/Buttons/CopyResponseButton.jsx @@ -1,12 +1,11 @@ -import React, { useContext, useState } from 'react'; -import OverlayImageButton from './OverlayImageButton'; - -import copyImage from '../../assets/img/copy.svg'; -import { ResponseContext } from '../Contexts/ResponseContext'; +import React, { useState } from 'react'; import { OverlayTrigger, Tooltip } from 'react-bootstrap'; +import ImageButton from './ImageButton'; +import { useQueryBuilderContext } from 'app/pages/querybuilder/Components/Contexts/QueryBuilderProvider'; +import copyImage from 'app/pages/querybuilder/assets/img/copy.svg'; export default function CopyResponseButton() { - const { response } = useContext(ResponseContext); + const response = useQueryBuilderContext((ctx) => ctx.http.response); const [show, setShow] = useState(false); const handleCopy = () => { @@ -20,18 +19,16 @@ export default function CopyResponseButton() { return ( <OverlayTrigger placement="left-end" - show={show} overlay={ - <Tooltip id="copy-tooltip">Response copied to clipboard</Tooltip> + <Tooltip>{show ? 'Response copied to clipboard' : 'Copy'}</Tooltip> } > <span> - <OverlayImageButton + <ImageButton className="intro-copy" image={copyImage} - height="30" - width="30" - tooltip="Copy" + height={30} + width={30} onClick={handleCopy} /> </span> diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/DownloadJSONButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/DownloadJSONButton.jsx index 7ec6683afa3..b89e3f14b3b 100644 --- a/client/js/app/src/app/pages/querybuilder/Components/Buttons/DownloadJSONButton.jsx +++ b/client/js/app/src/app/pages/querybuilder/Components/Buttons/DownloadJSONButton.jsx @@ -1,44 +1,31 @@ -import React, { useContext } from 'react'; -import { ResponseContext } from '../Contexts/ResponseContext'; -import transform from '../../TransformVespaTrace'; -import SimpleButton from './SimpleButton'; - -export default function DownloadJSONButton({ children }) { - const { response } = useContext(ResponseContext); - - const transformResponse = (response) => { - return transform(response); - }; +import React from 'react'; +import transform from 'app/pages/querybuilder/TransformVespaTrace'; +export default function DownloadJSONButton({ children, response }) { const handleClick = () => { - if (response != '') { - let transformedResponse = JSON.stringify( - transformResponse(JSON.parse(response), undefined, '\t') - ); - // copied from safakeskinĀ“s answer on SO, link: https://stackoverflow.com/questions/55613438/reactwrite-to-json-file-or-export-download-no-server - const fileName = 'vespa-response'; - const blob = new Blob([transformedResponse], { - type: 'application/json', - }); - const href = URL.createObjectURL(blob); + let content; + try { + content = JSON.stringify(transform(JSON.parse(response), null, 4)); + } catch (error) { + alert(`Failed to transform response to Jaeger format: ${error}`); // TODO: Change to toast + return; + } - // create "a" HTLM element with href to file - const link = document.createElement('a'); - link.href = href; - link.download = fileName + '.json'; - document.body.appendChild(link); - link.click(); + // copied from safakeskinĀ“s answer on SO, link: https://stackoverflow.com/questions/55613438/reactwrite-to-json-file-or-export-download-no-server + const blob = new Blob([content], { type: 'application/json' }); + const href = URL.createObjectURL(blob); - // clean up "a" element & remove ObjectURL - document.body.removeChild(link); - URL.revokeObjectURL(href); + // create "a" HTML element with href to file + const link = document.createElement('a'); + link.href = href; + link.download = 'vespa-response.json'; + document.body.appendChild(link); + link.click(); - // open Jaeger in a new tab - window.open('http://localhost:16686/search', '__blank'); - } else { - alert('Response was empty'); - } + // clean up "a" element & remove ObjectURL + document.body.removeChild(link); + URL.revokeObjectURL(href); }; - return <SimpleButton onClick={handleClick}>{children}</SimpleButton>; + return <button onClick={handleClick}>{children}</button>; } diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/ImageButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/ImageButton.jsx index f620146bea5..711229d82cd 100644 --- a/client/js/app/src/app/pages/querybuilder/Components/Buttons/ImageButton.jsx +++ b/client/js/app/src/app/pages/querybuilder/Components/Buttons/ImageButton.jsx @@ -4,14 +4,13 @@ export default function ImageButton({ onClick, children, className, - id, image, - height = '15', - width = '15', + height = 15, + width = 15, style, }) { return ( - <button id={id} className={className} onClick={onClick}> + <button className={className} onClick={onClick}> <img src={image} height={height} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/OverlayImageButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/OverlayImageButton.jsx deleted file mode 100644 index 788d88fd0e6..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Buttons/OverlayImageButton.jsx +++ /dev/null @@ -1,38 +0,0 @@ -import React from 'react'; -import OverlayTrigger from 'react-bootstrap/OverlayTrigger'; -import Tooltip from 'react-bootstrap/Tooltip'; -import ImageButton from './ImageButton'; - -export default function OverlayImageButton({ - onClick, - children, - className, - id, - image, - height = '15', - width = '15', - style, - tooltip, -}) { - return ( - <OverlayTrigger - placement="right" - delay={{ show: 250, hide: 400 }} - overlay={<Tooltip id="button-tooltip">{tooltip}</Tooltip>} - > - <span> - <ImageButton - id={id} - className={className} - image={image} - height={height} - width={width} - style={style} - onClick={onClick} - > - {children} - </ImageButton> - </span> - </OverlayTrigger> - ); -} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/PasteJSONButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/PasteJSONButton.jsx index df380c62fa1..f01f5b177a5 100644 --- a/client/js/app/src/app/pages/querybuilder/Components/Buttons/PasteJSONButton.jsx +++ b/client/js/app/src/app/pages/querybuilder/Components/Buttons/PasteJSONButton.jsx @@ -1,15 +1,14 @@ -import React, { useContext, useState } from 'react'; -import ImageButton from './ImageButton'; +import React, { useState } from 'react'; import pasteImage from '../../assets/img/paste.svg'; -import { QueryInputContext } from '../Contexts/QueryInputContext'; +import ImageButton from './ImageButton'; +import { + ACTION, + dispatch, +} from 'app/pages/querybuilder/Components/Contexts/QueryBuilderProvider'; export default function PasteJSONButton() { - const { setInputs, setId, levelZeroParameters, childMap } = - useContext(QueryInputContext); const [paste, setPaste] = useState(false); - //TODO: fix that the second-level dropdowns do not get set properly when pasting a JSON query - const handleClick = () => { setPaste(true); window.addEventListener('paste', handlePaste); @@ -21,69 +20,15 @@ export default function PasteJSONButton() { e.stopPropagation(); e.preventDefault(); const pastedData = e.clipboardData.getData('text'); - alert('Converting JSON: \n\n ' + pastedData); window.removeEventListener('paste', handlePaste); - convertPastedJSON(pastedData); - }; - - const convertPastedJSON = (pastedData) => { - try { - var json = JSON.parse(pastedData); - const newInputs = buildFromJSON(json, 2); - setInputs(newInputs); - } catch (error) { - console.log(error); - alert('Could not parse JSON, with error-message: \n\n' + error.message); - } - }; - - const buildFromJSON = (json, id, parentTypeof) => { - let newInputs = []; - let keys = Object.keys(json); - for (let i = 0; i < keys.length; i++) { - let childId = 1; - let newInput = { id: `${id}`, type: keys[i] }; - //If the value for the key is a child object - if (typeof json[keys[i]] === 'object') { - newInput['typeof'] = 'Parent'; - newInput['input'] = ''; - newInput['hasChildren'] = true; - // Construct the id of the correct pattern - let tempId = id + '.' + childId; - childId += 1; - let type; - if (id.length > 1) { - //Used to get the correct value from childMap - type = parentTypeof + '_' + keys[i]; - } else { - type = keys[i]; - } - newInput['children'] = buildFromJSON(json[keys[i]], tempId, type); - } else { - if (id.length > 1) { - const choices = childMap[parentTypeof]; - newInput['typeof'] = choices[keys[i]].type; - } else { - newInput['typeof'] = levelZeroParameters[keys[i]].type; - } - newInput['input'] = json[keys[i]]; - newInput['hasChildren'] = false; - newInput['children'] = []; - } - id += 1; - newInputs.push(newInput); - } - setId(id); - return newInputs; + dispatch(ACTION.SET_QUERY, pastedData); }; return ( <> <ImageButton - id="pasteJSON" className="pasteJSON" image={pasteImage} - //style={{ marginTop: '-2px', marginRight: '3px' }} onClick={handleClick} > {paste ? 'Press CMD + V' : 'Paste JSON'} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/ShowQueryButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/ShowQueryButton.jsx deleted file mode 100644 index 789fc387b38..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Buttons/ShowQueryButton.jsx +++ /dev/null @@ -1,29 +0,0 @@ -import React, { useContext, useState } from 'react'; -import { QueryContext } from '../Contexts/QueryContext'; -import SimpleButton from './SimpleButton'; - -export default function ShowQueryButton() { - const { query, showQuery, setShowQuery } = useContext(QueryContext); - - const handleClick = () => { - setShowQuery(!showQuery); - }; - - return ( - <> - <SimpleButton className="showJSON" onClick={handleClick}> - Show query JSON - </SimpleButton> - {showQuery && ( - <textarea - id="jsonquery" - className="responsebox" - readOnly - cols="70" - rows="15" - value={query} - ></textarea> - )} - </> - ); -} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Buttons/SimpleButton.jsx b/client/js/app/src/app/pages/querybuilder/Components/Buttons/SimpleButton.jsx deleted file mode 100644 index a153c9577e4..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Buttons/SimpleButton.jsx +++ /dev/null @@ -1,9 +0,0 @@ -import React from 'react'; - -export default function SimpleButton({ onClick, children, className, id }) { - return ( - <button id={id} className={className} onClick={onClick}> - {children} - </button> - ); -} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryBuilderProvider.jsx b/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryBuilderProvider.jsx new file mode 100644 index 00000000000..2eebbb9b8b5 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryBuilderProvider.jsx @@ -0,0 +1,146 @@ +import React, { useReducer } from 'react'; +import { createContext, useContextSelector } from 'use-context-selector'; +import { cloneDeep, last } from 'lodash'; +import parameters from 'app/pages/querybuilder/parameters'; + +let _dispatch; +const root = { type: { children: parameters } }; +const context = createContext(null); + +export const ACTION = Object.freeze({ + SET_QUERY: 0, + SET_HTTP: 1, + + INPUT_ADD: 10, + INPUT_UPDATE: 11, + INPUT_REMOVE: 12, +}); + +function inputsToJson(inputs) { + return Object.fromEntries( + inputs.map(({ children, input, type: { name, type } }) => [ + name, + children ? inputsToJson(children) : parseInput(input, type), + ]) + ); +} + +function jsonToInputs(json, parent) { + return Object.entries(json).map(([key, value], i) => { + const node = { + id: parent.id ? `${parent.id}.${i}` : i.toString(), + type: parent.type.children[key], + parent, + }; + if (typeof value === 'object') { + node.input = ''; + node.children = jsonToInputs(value, node); + } else node.input = value.toString(); + return node; + }); +} + +function parseInput(input, type) { + if (type === 'Integer' || type === 'Long') return parseInt(input); + if (type === 'Float') return parseFloat(input); + if (type === 'Boolean') return input.toLowerCase() === 'true'; + return input; +} + +function inputAdd(query, { id: parentId, type: typeName }) { + const inputs = cloneDeep(query.children); + const parent = parentId ? findInput(inputs, parentId) : query; + + const nextId = + parseInt(last(last(parent.children)?.id?.split('.')) ?? -1) + 1; + const id = parentId ? `${parentId}.${nextId}` : nextId.toString(); + const type = parent.type.children[typeName]; + + parent.children.push( + Object.assign( + { id, input: '', type, parent }, + type.children && { children: [] } + ) + ); + return { ...query, children: inputs }; +} + +function inputUpdate(query, { id, ...props }) { + const keys = Object.keys(props); + if (keys.length !== 1) + throw new Error(`Expected to update exactly 1 input prop, got: ${keys}`); + if (!['input', 'type'].includes(keys[0])) + throw new Error(`Cannot update key ${keys[0]}`); + + const inputs = cloneDeep(query.children); + const node = Object.assign(findInput(inputs, id), props); + if (node.type.children) node.children = []; + else delete node.children; + return { ...query, children: inputs }; +} + +function findInput(inputs, id, Delete = false) { + let end = -1; + while ((end = id.indexOf('.', end + 1)) > 0) + inputs = inputs.find((input) => input.id === id.substring(0, end)).children; + const index = inputs.findIndex((input) => input.id === id); + return Delete ? inputs.splice(index, 1)[0] : inputs[index]; +} + +function reducer(state, action) { + const result = preReducer(state, action); + if (state.query.children !== result.query.children) { + const json = inputsToJson(result.query.children); + result.query.input = JSON.stringify(json, null, 4); + } + + return result; +} +function preReducer(state, { action, data }) { + switch (action) { + case ACTION.SET_QUERY: { + try { + const children = jsonToInputs(JSON.parse(data), root); + return { ...state, query: { ...root, children } }; + } catch (error) { + alert(`Failed to parse query: ${error}`); // TODO: Change to toast + return state; + } + } + case ACTION.SET_HTTP: + return { ...state, http: data }; + + case ACTION.INPUT_ADD: + return { ...state, query: inputAdd(state.query, data) }; + case ACTION.INPUT_UPDATE: + return { ...state, query: inputUpdate(state.query, data) }; + case ACTION.INPUT_REMOVE: { + const inputs = cloneDeep(state.query.children); + findInput(inputs, data, true); + return { ...state, query: { ...state.query, children: inputs } }; + } + + default: + throw new Error(`Unknown action ${action}`); + } +} + +export function QueryBuilderProvider({ children }) { + const [value, dispatch] = useReducer( + reducer, + { http: {}, query: { ...root, input: '', children: [] } }, + (s) => reducer(s, { action: ACTION.SET_QUERY, data: '{"yql":""}' }) + ); + _dispatch = dispatch; + + return <context.Provider value={value}>{children}</context.Provider>; +} + +export function useQueryBuilderContext(selector) { + const func = typeof selector === 'string' ? (c) => c[selector] : selector; + return useContextSelector(context, func); +} + +export function dispatch(action, data) { + _dispatch({ action, data }); +} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryContext.jsx b/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryContext.jsx deleted file mode 100644 index 00644c21d41..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryContext.jsx +++ /dev/null @@ -1,14 +0,0 @@ -import React, { createContext, useState } from 'react'; - -export const QueryContext = createContext(); - -export const QueryProvider = (prop) => { - const [query, setQuery] = useState(''); - const [showQuery, setShowQuery] = useState(false); - - return ( - <QueryContext.Provider value={{ query, setQuery, showQuery, setShowQuery }}> - {prop.children} - </QueryContext.Provider> - ); -}; diff --git a/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryInputContext.jsx b/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryInputContext.jsx deleted file mode 100644 index bc21ea81d9a..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Contexts/QueryInputContext.jsx +++ /dev/null @@ -1,160 +0,0 @@ -import React, { useState, createContext } from 'react'; - -export const QueryInputContext = createContext(); - -export const QueryInputProvider = (prop) => { - // This is the id of the newest QueryInput, gets updated each time a new one is added - const [id, setId] = useState(1); - - // These are the methods that can be chosen in a QueryInput - const levelZeroParameters = { - yql: { name: 'yql', type: 'String', hasChildren: false }, - hits: { name: 'hits', type: 'Integer', hasChildren: false }, - offset: { name: 'offset', type: 'Integer', hasChildren: false }, - queryProfile: { name: 'queryProfile', type: 'String', hasChildren: false }, - noCache: { name: 'noCache', type: 'Boolean', hasChildren: false }, - groupingSessionCache: { - name: 'groupingSessionCache', - type: 'Boolean', - hasChildren: false, - }, - searchChain: { name: 'searchChain', type: 'String', hasChildren: false }, - timeout: { name: 'timeout', type: 'Float', hasChildren: false }, - trace: { name: 'trace', type: 'Parent', hasChildren: true }, - tracelevel: { name: 'tracelevel', type: 'Parent', hasChildren: true }, - traceLevel: { name: 'traceLevel', type: 'Integer', hasChildren: false }, - explainLevel: { name: 'explainLevel', type: 'Integer', hasChildren: false }, - explainlevel: { name: 'explainlevel', type: 'Integer', hasChildren: false }, - model: { name: 'model', type: 'Parent', hasChildren: true }, - ranking: { name: 'ranking', type: 'Parent', hasChildren: true }, - collapse: { name: 'collapse', type: 'Parent', hasChildren: true }, - collapsesize: { name: 'collapsesize', type: 'Integer', hasChildren: false }, - collapsefield: { - name: 'collapsefield', - type: 'String', - hasChildren: false, - }, - presentation: { name: 'presentation', type: 'Parent', hasChildren: true }, - pos: { name: 'pos', type: 'Parent', hasChildren: true }, - streaming: { name: 'streaming', type: 'Parent', hasChildren: true }, - rules: { name: 'rules', type: 'Parent', hasChildren: true }, - recall: { name: 'recall', type: 'List', hasChildren: false }, - user: { name: 'user', type: 'String', hasChildren: false }, - metrics: { name: 'metrics', type: 'Parent', hasChildren: true }, - }; - - // Children of the levelZeroParameters that have child attributes - const childMap = { - collapse: { - summary: { name: 'summary', type: 'String', hasChildren: false }, - }, - metrics: { - ignore: { name: 'ignore', type: 'Boolean', hasChildren: false }, - }, - model: { - defaultIndex: { - name: 'defaultIndex', - type: 'String', - hasChildren: false, - }, - encoding: { name: 'encoding', type: 'String', hasChildren: false }, - language: { name: 'language', type: 'String', hasChildren: false }, - queryString: { name: 'queryString', type: 'String', hasChildren: false }, - restrict: { name: 'restrict', type: 'List', hasChildren: false }, - searchPath: { name: 'searchPath', type: 'String', hasChildren: false }, - sources: { name: 'sources', type: 'List', hasChildren: false }, - type: { name: 'type', type: 'String', hasChildren: false }, - }, - pos: { - ll: { name: 'll', type: 'String', hasChildren: false }, - radius: { name: 'radius', type: 'String', hasChildren: false }, - bb: { name: 'bb', type: 'List', hasChildren: false }, - attribute: { name: 'attribute', type: 'String', hasChildren: false }, - }, - presentation: { - bolding: { name: 'bolding', type: 'Boolean', hasChildren: false }, - format: { name: 'format', type: 'String', hasChildren: false }, - summary: { name: 'summary', type: 'String', hasChildren: false }, - template: { name: 'template', type: 'String', hasChildren: false }, - timing: { name: 'timing', type: 'Boolean', hasChildren: false }, - }, - ranking: { - location: { name: 'location', type: 'String', hasChildren: false }, - features: { name: 'features', type: 'String', hasChildren: false }, - listFeatures: { - name: 'listFeatures', - type: 'Boolean', - hasChildren: false, - }, - profile: { name: 'profile', type: 'String', hasChildren: false }, - properties: { name: 'properties', type: 'String', hasChildren: false }, - sorting: { name: 'sorting', type: 'String', hasChildren: false }, - freshness: { name: 'freshness', type: 'String', hasChildren: false }, - queryCache: { name: 'queryCache', type: 'Boolean', hasChildren: false }, - matchPhase: { name: 'matchPhase', type: 'Parent', hasChildren: true }, - }, - ranking_matchPhase: { - maxHits: { name: 'maxHits', type: 'Long', hasChildren: false }, - attribute: { name: 'attribute', type: 'String', hasChildren: false }, - ascending: { name: 'ascending', type: 'Boolean', hasChildren: false }, - diversity: { name: 'diversity', type: 'Parent', hasChildren: true }, - }, - ranking_matchPhase_diversity: { - attribute: { name: 'attribute', type: 'String', hasChildren: false }, - minGroups: { name: 'minGroups', type: 'Long', hasChildren: false }, - }, - rules: { - off: { name: 'off', type: 'Boolean', hasChildren: false }, - rulebase: { name: 'rulebase', type: 'String', hasChildren: false }, - }, - streaming: { - userid: { name: 'userid', type: 'Integer', hasChildren: false }, - groupname: { name: 'groupname', type: 'String', hasChildren: false }, - selection: { name: 'selection', type: 'String', hasChildren: false }, - priority: { name: 'priority', type: 'String', hasChildren: false }, - maxbucketspervisitor: { - name: 'maxbucketspervisitor', - type: 'Integer', - hasChildren: false, - }, - }, - trace: { - timestamps: { name: 'timestamps', type: 'Boolean', hasChildren: false }, - }, - tracelevel: { - rules: { name: 'rules', type: 'Integer', hasChildren: false }, - }, - }; - - const firstChoice = levelZeroParameters[Object.keys(levelZeroParameters)[0]]; - - const [inputs, setInputs] = useState([ - { - id: '1', - type: firstChoice.name, - typeof: firstChoice.type, - input: '', - hasChildren: false, - children: [], - }, - ]); - - const [selectedItems, setSelectedItems] = useState([]); - - return ( - <QueryInputContext.Provider - value={{ - inputs, - setInputs, - id, - setId, - levelZeroParameters, - childMap, - selectedItems, - setSelectedItems, - }} - > - {prop.children} - </QueryInputContext.Provider> - ); -}; diff --git a/client/js/app/src/app/pages/querybuilder/Components/Contexts/ResponseContext.jsx b/client/js/app/src/app/pages/querybuilder/Components/Contexts/ResponseContext.jsx deleted file mode 100644 index 54f5fc955fd..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Contexts/ResponseContext.jsx +++ /dev/null @@ -1,13 +0,0 @@ -import React, { createContext, useState } from 'react'; - -export const ResponseContext = createContext(); - -export const ResponseProvider = (prop) => { - const [response, setResponse] = useState(''); - - return ( - <ResponseContext.Provider value={{ response, setResponse }}> - {prop.children} - </ResponseContext.Provider> - ); -}; diff --git a/client/js/app/src/app/pages/querybuilder/Components/Navigation/CustomNavbar.jsx b/client/js/app/src/app/pages/querybuilder/Components/Navigation/CustomNavbar.jsx deleted file mode 100644 index 6ffeae4caed..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Navigation/CustomNavbar.jsx +++ /dev/null @@ -1,37 +0,0 @@ -import React from 'react'; -import { Navbar, Container, Nav } from 'react-bootstrap'; - -function CustomNavbar() { - return ( - <Navbar collapseOnSelect variant="default" expand="sm"> - <Container> - <Navbar.Brand href="https://vespa.ai"> - Vespa. Big data. Real time. - </Navbar.Brand> - <Navbar.Toggle aria-controls="responsive-navbar-nav" /> - <Navbar.Collapse id="responsive-navbar-nav"> - <Nav> - <Nav.Link href="https://blog.vespa.ai/">Blog</Nav.Link> - <Nav.Link variant="link" href="https://twitter.com/vespaengine"> - Twitter - </Nav.Link> - <Nav.Link variant="link" href="https://docs.vespa.ai"> - Docs - </Nav.Link> - <Nav.Link variant="link" href="https://github.com/vespa-engine"> - GitHub - </Nav.Link> - <Nav.Link - variant="link" - href="https://docs.vespa.ai/en/getting-started.html" - > - Get Started Now - </Nav.Link> - </Nav> - </Navbar.Collapse> - </Container> - </Navbar> - ); -} - -export default CustomNavbar; diff --git a/client/js/app/src/app/pages/querybuilder/Components/Navigation/Footer.jsx b/client/js/app/src/app/pages/querybuilder/Components/Navigation/Footer.jsx deleted file mode 100644 index b110bce943d..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Navigation/Footer.jsx +++ /dev/null @@ -1,68 +0,0 @@ -import React from 'react'; -import { Table } from 'react-bootstrap'; - -export default function Footer() { - return ( - <footer> - <Table borderless size="sm"> - <thead className="footer-title"> - <tr> - <th>Resources</th> - <th>Contact</th> - <th>Community</th> - </tr> - </thead> - <tbody> - <tr> - <th> - <a href="https://docs.vespa.ai/en/vespa-quick-start.html"> - Getting Started - </a> - </th> - <th> - <a href="https://twitter.com/vespaengine">Twitter</a> - </th> - <th> - <a href="https://github.com/vespa-engine/vespa/blob/master/CONTRIBUTING.md"> - Contributing - </a> - </th> - </tr> - <tr> - <th> - <a href="https://docs.vespa.ai">Documentation</a> - </th> - <th> - <a href="mailto:info@vespa.ai">info@vespa.ai</a> - </th> - <th> - <a href="https://stackoverflow.com/questions/tagged/vespa"> - Stack Overflow - </a> - </th> - </tr> - <tr> - <th> - <a href="https://github.com/vespa-engine/vespa">Open source</a> - </th> - <th> - <a href="https://github.com/vespa-engine/vespa/issues">Issues</a> - </th> - <th> - <a href="https://gitter.im/vespa-engine/Lobby">Gitter</a> - </th> - </tr> - </tbody> - </Table> - <div className="credits"> - <span>Copyright Yahoo</span> - Licensed under{' '} - <a href="https://github.com/vespa-engine/vespa/blob/master/LICENSE"> - Apache License 2.0 - </a> - , <a href="https://github.com/y7kim/agency-jekyll-theme">Theme</a> by - Rick K. - </div> - </footer> - ); -} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/Info.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/Info.jsx deleted file mode 100644 index d921c53a602..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Text/Info.jsx +++ /dev/null @@ -1,37 +0,0 @@ -import React from 'react'; -import { OverlayTrigger, Popover } from 'react-bootstrap'; -import image from '../../assets/img/information.svg'; - -export default function Info({ - id, - className = 'tip', - height = 15, - width = 15, -}) { - const popover = ( - <Popover id={`inf${id}`}> - <Popover.Header as="h3">Popover right</Popover.Header> - <Popover.Body>Content</Popover.Body> - </Popover> - ); - - return ( - <> - <OverlayTrigger - placement="right" - delay={{ show: 250, hide: 400 }} - overlay={popover} - > - <span> - <img - src={image} - height={height} - width={width} - className="information" - alt="Missing" - /> - </span> - </OverlayTrigger> - </> - ); -} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/QueryDropDownForm.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/QueryDropDownForm.jsx deleted file mode 100644 index 88bd545282a..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Text/QueryDropDownForm.jsx +++ /dev/null @@ -1,75 +0,0 @@ -import React, { useContext, useState } from 'react'; -import { QueryInputContext } from '../Contexts/QueryInputContext'; -import SimpleDropDownForm from './SimpleDropDownForm'; - -export default function QueryDropdownForm({ - choices, - id, - child = false, - initial, -}) { - const { - inputs, - setInputs, - levelZeroParameters, - childMap, - selectedItems, - setSelectedItems, - } = useContext(QueryInputContext); - const [choice, setChoice] = useState(); - - /** - * Update the state of inputs to reflect the method chosen from the dropdown. - * If the prevoiusly chosen method had children they are removed. - * @param {Event} e Event containing the new type. - */ - const updateType = (e) => { - e.preventDefault(); - const newType = e.target.value; - const newInputs = inputs.slice(); - let currentId = id.substring(0, 1); - let index = newInputs.findIndex((element) => element.id === currentId); - if (child) { - let parentTypes = newInputs[index].type; - let children = newInputs[index].children; - let childChoices = childMap[parentTypes]; - //TODO: try to rafactor this loop into a separate function that can be - //used everywhere in the code similar loops are used - for (let i = 3; i < id.length; i += 2) { - currentId = id.substring(0, i); - index = children.findIndex((element) => element.id === currentId); - let child = children[index]; - parentTypes = parentTypes + '_' + child.type; - childChoices = childMap[parentTypes]; - children = child.children; - } - index = children.findIndex((element) => element.id === id); - children[index].type = newType; - children[index].hasChildren = childChoices[newType].hasChildren; - children[index].children = []; - children[index].typeof = childChoices[newType].type; - setSelectedItems([...selectedItems, newType]); - } else { - newInputs[index].type = newType; - let hasChildren = levelZeroParameters[newType].hasChildren; - newInputs[index].hasChildren = hasChildren; - newInputs[index].children = []; - newInputs[index].typeof = levelZeroParameters[newType].type; - setSelectedItems([...selectedItems, newType]); - } - setInputs(newInputs); - setChoice(newType); - }; - - //TODO: do not display options that have been chosen - - return ( - <SimpleDropDownForm - id={id} - onChange={updateType} - choices={choices} - value={choice} - initial={initial} - ></SimpleDropDownForm> - ); -} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInput.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInput.jsx index bd41edd6008..5f5a9fdbd89 100644 --- a/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInput.jsx +++ b/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInput.jsx @@ -1,79 +1,91 @@ -import React, { useContext } from 'react'; -import SimpleButton from '../Buttons/SimpleButton'; -import SimpleForm from './SimpleForm'; +import React from 'react'; import { OverlayTrigger, Tooltip } from 'react-bootstrap'; -import { QueryInputContext } from '../Contexts/QueryInputContext'; -import QueryDropdownForm from './QueryDropDownForm'; -import AddPropertyButton from '../Buttons/AddPropertyButton'; -import QueryInputChild from './QueryInputChild'; +import SimpleDropDownForm from 'app/pages/querybuilder/Components/Text/SimpleDropDownForm'; +import { + ACTION, + dispatch, + useQueryBuilderContext, +} from 'app/pages/querybuilder/Components/Contexts/QueryBuilderProvider'; export default function QueryInput() { - const { inputs, setInputs, levelZeroParameters } = - useContext(QueryInputContext); - - function removeRow(id) { - const newList = inputs.filter((item) => item.id !== id); - setInputs(newList); - } - - const updateInput = (e) => { - e.preventDefault(); - const fid = e.target.id.replace('v', ''); - const newInputs = inputs.slice(); - const index = newInputs.findIndex((element) => element.id === fid); - newInputs[index].input = e.target.value; - setInputs(newInputs); - }; + const { children, type } = useQueryBuilderContext('query'); + return <Inputs type={type.children} inputs={children} />; +} - const setPlaceholder = (id) => { - try { - const index = inputs.findIndex((element) => element.id === id); - return inputs[index].typeof; - } catch (error) { - console.log(error); - } - }; +function Inputs({ id, type, inputs }) { + const usedTypes = inputs.map(({ type }) => type.name); + const remainingTypes = Object.fromEntries( + Object.entries(type).filter(([name]) => !usedTypes.includes(name)) + ); + const firstRemaining = Object.keys(remainingTypes)[0]; + return ( + <> + {inputs.map(({ id, input, type, children }) => ( + <Input + key={id} + types={remainingTypes} + {...{ id, input, type, children }} + /> + ))} + {firstRemaining && <AddPropertyButton id={id} type={firstRemaining} />} + </> + ); +} - const inputList = inputs.map((value) => { - return ( - <div key={value.id + value.typeof} id={value.id} className="queryinput"> - <QueryDropdownForm - choices={levelZeroParameters} - id={value.id} - initial={value.type} +function Input({ id, input, types, type, children }) { + return ( + <div className="queryinput"> + <SimpleDropDownForm + onChange={({ target }) => + dispatch(ACTION.INPUT_UPDATE, { + id, + type: types[target.value], + }) + } + options={{ [type.name]: type, ...types }} + value={type.name} + /> + {children ? ( + <Inputs id={id} type={type.children} inputs={children} /> + ) : ( + <input + size="30" + onChange={({ target }) => + dispatch(ACTION.INPUT_UPDATE, { + id, + input: target.value, + }) + } + placeholder={type.type} + value={input} /> - {value.hasChildren ? ( - <> - <AddPropertyButton id={value.id} /> - <QueryInputChild id={value.id} /> - </> - ) : ( - <SimpleForm - id={`v${value.id}`} - size="30" - onChange={updateInput} - placeholder={setPlaceholder(value.id)} - initial={value.input} - /> - )} - <OverlayTrigger - placement="right" - delay={{ show: 250, hide: 400 }} - overlay={<Tooltip id="button-tooltip">Remove row</Tooltip>} - > - <span> - <SimpleButton - id={`b${value.id}`} - className="removeRow" - onClick={() => removeRow(value.id)} - children="-" - ></SimpleButton> - </span> - </OverlayTrigger> - <br /> - </div> - ); - }); + )} + <OverlayTrigger + placement="right" + delay={{ show: 250, hide: 400 }} + overlay={<Tooltip id="button-tooltip">Remove row</Tooltip>} + > + <span> + <button + className="removeRow" + onClick={() => dispatch(ACTION.INPUT_REMOVE, id)} + > + - + </button> + </span> + </OverlayTrigger> + <br /> + </div> + ); +} - return <>{inputList}</>; +function AddPropertyButton({ id, type }) { + return ( + <button + className="addpropsbutton" + onClick={() => dispatch(ACTION.INPUT_ADD, { id, type })} + > + + Add property + </button> + ); } diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInputChild.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInputChild.jsx deleted file mode 100644 index 4ab2a074214..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Text/QueryInputChild.jsx +++ /dev/null @@ -1,201 +0,0 @@ -import React, { useContext } from 'react'; -import AddPropertyButton from '../Buttons/AddPropertyButton'; -import { QueryInputContext } from '../Contexts/QueryInputContext'; -import QueryDropdownForm from './QueryDropDownForm'; -import SimpleForm from './SimpleForm'; -import { OverlayTrigger, Tooltip } from 'react-bootstrap'; -import SimpleButton from '../Buttons/SimpleButton'; - -export default function QueryInputChild({ id }) { - const { inputs, setInputs, childMap } = useContext(QueryInputContext); - - let index = inputs.findIndex((element) => element.id === id); - let childArray = inputs[index].children; - let currentTypes = inputs[index].type; - - /** - * Update the state of inputs to reflect what is written into the form. - * @param {Event} e Event containing the new input. - */ - const updateInput = (e) => { - e.preventDefault(); - let newInputs = inputs.slice(); - let iterId = e.target.id.replace('v', ''); - let currentId = iterId.substring(0, 1); - let index = newInputs.findIndex((element) => element.id === currentId); - let children = newInputs[index].children; - const traversedChildren = traverseChildren(iterId, children, ''); - children = traversedChildren.children; - index = children.findIndex((element) => element.id === iterId); - children[index].input = e.target.value; - setInputs(newInputs); - }; - - /** - * Returns a placeholder text for a SimpleForm component. - * @param {String} id The id of the SimpleForm component. - * @returns {String} The placeholder text - */ - const setPlaceHolder = (id) => { - let currentId = id.substring(0, 1); - let index = inputs.findIndex((element) => element.id === currentId); - let combinedType = inputs[index].type; - let children = inputs[index].children; - if (id.length > 3) { - const traversedChildren = traverseChildren(id, children, combinedType); - combinedType = traversedChildren.combinedType; - children = traversedChildren.children; - const currentChoice = childMap[combinedType]; - index = children.findIndex((element) => element.id === id); - combinedType = children[index].type; - return currentChoice[combinedType].type; - } else { - const currentChoice = childMap[combinedType]; - index = children.findIndex((element) => element.id === id); - combinedType = children[index].type; - return currentChoice[combinedType].type; - } - }; - - /** - * Removes the row with the provided id. - * @param {String} id Id of row. - */ - const removeRow = (id) => { - let newInputs = inputs.slice(); - let currentId = id.substring(0, 1); - let index = newInputs.findIndex((element) => element.id === currentId); - let children = newInputs[index].children; - const traversedChildren = traverseChildren(id, children, ''); - index = traversedChildren.children.findIndex((element) => element === id); - traversedChildren.children.splice(index, 1); - setInputs(newInputs); - }; - - /** - * Traverses the children until a child with the provided id is reached. - * @param {String} id Id of the innermost child. - * @param {Array} children Array containing serveral child objects. - * @param {String} combinedType The combined type of all traversed children - * @returns {Object} An object containing the children of the child with the provided id and the combined type. - */ - function traverseChildren(id, children, combinedType) { - let currentId; - let index; - for (let i = 3; i < id.length; i += 2) { - currentId = id.substring(0, i); - index = children.findIndex((element) => element.id === currentId); - combinedType = combinedType + '_' + children[index].type; - children = children[index].children; - } - return { children: children, combinedType: combinedType }; - } - - const inputList = childArray.map((child) => { - return ( - <div key={child.id} id={child.id}> - { - //child.id == '4.1' && console.log(child.type) - } - <QueryDropdownForm - choices={childMap[currentTypes]} - id={child.id} - child={true} - inital={child.type} - /> - {child.hasChildren ? ( - <> - <AddPropertyButton id={child.id} /> - </> - ) : ( - <SimpleForm - id={`v${child.id}`} - size="30" - onChange={updateInput} - placeholder={setPlaceHolder(child.id)} - inital={child.input} - /> - )} - <OverlayTrigger - placement="right" - delay={{ show: 250, hide: 400 }} - overlay={<Tooltip id="button-tooltip">Remove row</Tooltip>} - > - <span> - <SimpleButton - id={`b${child.id}`} - className="removeRow" - onClick={() => removeRow(child.id)} - children="-" - ></SimpleButton> - </span> - </OverlayTrigger> - <br /> - <Child - type={currentTypes + '_' + child.type} - child={child} - onChange={updateInput} - placeholder={setPlaceHolder} - removeRow={removeRow} - /> - </div> - ); - }); - - return <>{inputList}</>; -} - -function Child({ child, type, onChange, placeholder, removeRow }) { - const { childMap } = useContext(QueryInputContext); - - const nestedChildren = (child.children || []).map((child) => { - return ( - <div key={child.id}> - <QueryDropdownForm - choices={childMap[type]} - id={child.id} - child={true} - initial={child.type} - /> - {child.hasChildren ? ( - <> - <AddPropertyButton id={child.id} /> - </> - ) : ( - <SimpleForm - id={`v${child.id}`} - size="30" - onChange={onChange} - placeholder={placeholder(child.id)} - initial={child.input} - /> - )} - <OverlayTrigger - placement="right" - delay={{ show: 250, hide: 400 }} - overlay={<Tooltip id="button-tooltip">Remove row</Tooltip>} - > - <span> - <SimpleButton - id={`b${child.id}`} - className="removeRow" - onClick={() => removeRow(child.id)} - children="-" - ></SimpleButton> - </span> - </OverlayTrigger> - <br /> - <Child - child={child} - id={child.id} - type={type + '_' + child.type} - onChange={onChange} - placeholder={placeholder} - removeRow={removeRow} - /> - </div> - ); - }); - - return <>{nestedChildren}</>; -} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/ResponseBox.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/ResponseBox.jsx deleted file mode 100644 index dac98271965..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Text/ResponseBox.jsx +++ /dev/null @@ -1,17 +0,0 @@ -import React, { useContext } from 'react'; -import { ResponseContext } from '../Contexts/ResponseContext'; - -export default function ResponseBox() { - const { response } = useContext(ResponseContext); - - return ( - <textarea - id="responsetext" - className="responsebox" - readOnly - cols="70" - rows="25" - value={response} - /> - ); -} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/SendQuery.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/SendQuery.jsx index a3714f27fb5..303bc8bfc83 100644 --- a/client/js/app/src/app/pages/querybuilder/Components/Text/SendQuery.jsx +++ b/client/js/app/src/app/pages/querybuilder/Components/Text/SendQuery.jsx @@ -1,24 +1,32 @@ -import React, { useContext, useEffect, useState } from 'react'; -import SimpleButton from '../Buttons/SimpleButton'; -import { QueryInputContext } from '../Contexts/QueryInputContext'; -import { ResponseContext } from '../Contexts/ResponseContext'; -import { QueryContext } from '../Contexts/QueryContext'; -import SimpleForm from './SimpleForm'; +import React, { useState } from 'react'; import SimpleDropDownForm from './SimpleDropDownForm'; +import { + ACTION, + dispatch, + useQueryBuilderContext, +} from 'app/pages/querybuilder/Components/Contexts/QueryBuilderProvider'; + +function send(method, url, query) { + dispatch(ACTION.SET_HTTP, { loading: true }); + fetch(url, { + method, + headers: { 'Content-Type': 'application/json;charset=utf-8' }, + body: query, + }) + .then((response) => response.json()) + .then((result) => + dispatch(ACTION.SET_HTTP, { + response: JSON.stringify(result, null, 4), + }) + ) + .catch((error) => dispatch(ACTION.SET_HTTP, { error })); +} export default function SendQuery() { - const { inputs } = useContext(QueryInputContext); - const { setResponse } = useContext(ResponseContext); - const { showQuery, setQuery } = useContext(QueryContext); - const messageMethods = { post: { name: 'POST' }, get: { name: 'GET' } }; const [method, setMethod] = useState(messageMethods.post.name); const [url, setUrl] = useState('http://localhost:8080/search/'); - - useEffect(() => { - const query = buildJSON(inputs, {}); - setQuery(JSON.stringify(query, undefined, 4)); - }, [showQuery]); + const query = useQueryBuilderContext((ctx) => ctx.query.input); const updateMethod = (e) => { e.preventDefault(); @@ -26,80 +34,23 @@ export default function SendQuery() { setMethod(newMethod); }; - function handleClick() { - const json = buildJSON(inputs, {}); - send(json); - } - - async function send(json) { - let responses = await fetch(url, { - method: method, - headers: { 'Content-Type': 'application/json;charset=utf-8' }, - body: JSON.stringify(json), - }); - if (responses.ok) { - let result = await responses.json(); - let resultObject = JSON.stringify(result, undefined, 4); - setResponse(resultObject); - } - } - - function buildJSON(inputs, json) { - let queryJson = json; - for (let i = 0; i < inputs.length; i++) { - let current = inputs[i]; - let key = current.type; - if (current.hasChildren) { - let child = {}; - child = buildJSON(current.children, child); - queryJson[key] = child; - } else { - queryJson[key] = parseInput(current.input, current.typeof); - } - } - return queryJson; - } - - function parseInput(input, type) { - switch (type) { - case 'Integer': - case 'Long': - return parseInt(input); - - case 'Float': - return parseFloat(input); - - case 'Boolean': - return input.toLowerCase() === 'true' ? true : false; - - default: - return input; - } - } - - const updateUrl = (e) => { - const newUrl = e.target.value; - setUrl(newUrl); - }; - return ( <> <SimpleDropDownForm - choices={messageMethods} - id="method" + options={messageMethods} + value={method} className="methodselector" onChange={updateMethod} /> - <SimpleForm - id="url" - className="textbox" - initial={url} + <input size="30" - onChange={updateUrl} + className="textbox" + value={url} + onChange={({ target }) => setUrl(target.value)} /> - <SimpleButton id="send" className="button" onClick={handleClick}> + <button className="button" onClick={() => send(method, url, query)}> Send - </SimpleButton> + </button> </> ); } diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleDropDownForm.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleDropDownForm.jsx index 94c6c01b619..99342a5ae81 100644 --- a/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleDropDownForm.jsx +++ b/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleDropDownForm.jsx @@ -1,44 +1,18 @@ -import React, { useContext, useEffect } from 'react'; -import { QueryInputContext } from '../Contexts/QueryInputContext'; +import React from 'react'; export default function SimpleDropDownForm({ - choices, - id, - className = 'input', - onChange, + options, value, - initial, + className = 'input', + ...props }) { - const { selectedItems } = useContext(QueryInputContext); - - //TODO: using the filtered list to render options results in dropdown not changing the displayed selection to what was actually selected. - let filtered = Object.keys(choices).filter( - (choice) => !selectedItems.includes(choice) - ); - useEffect(() => { - filtered = Object.keys(choices).filter( - (choice) => !selectedItems.includes(choice) - ); - }, [selectedItems]); - - const options = Object.keys(choices).map((choice) => { - return ( - <option className="options" key={choice} value={choices[choice].name}> - {choices[choice].name} - </option> - ); - }); - return ( - <form id={id}> - <select - className={className} - id={id} - defaultValue={initial} - onChange={onChange} - > - {options} - </select> - </form> + <select className={className} {...props} value={value}> + {Object.values(options).map(({ name }) => ( + <option className="options" key={name} value={name}> + {name} + </option> + ))} + </select> ); } diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleForm.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleForm.jsx deleted file mode 100644 index bb6aaa13529..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Text/SimpleForm.jsx +++ /dev/null @@ -1,34 +0,0 @@ -import React from 'react'; -import { useState } from 'react'; - -export default function SimpleForm({ - id, - className = 'propvalue', - initial, - size = '20', - onChange, - placeholder, -}) { - SimpleForm.defaultProps = { - onChange: handleChange, - }; - const [input, setValue] = useState(initial); - - function handleChange(e) { - setValue(e.target.value); - } - - return ( - <form className={className} id={id}> - <input - size={size} - type="text" - id={id} - className={className} - defaultValue={initial} - onChange={onChange} - placeholder={placeholder} - /> - </form> - ); -} diff --git a/client/js/app/src/app/pages/querybuilder/Components/Text/TextBox.jsx b/client/js/app/src/app/pages/querybuilder/Components/Text/TextBox.jsx deleted file mode 100644 index 022b250da7c..00000000000 --- a/client/js/app/src/app/pages/querybuilder/Components/Text/TextBox.jsx +++ /dev/null @@ -1,9 +0,0 @@ -import React from 'react'; - -export default function TextBox({ id, className, children }) { - return ( - <p className={className} id={id}> - {children} - </p> - ); -} diff --git a/client/js/app/src/app/pages/querybuilder/parameters.jsx b/client/js/app/src/app/pages/querybuilder/parameters.jsx new file mode 100644 index 00000000000..6557cfc0ea0 --- /dev/null +++ b/client/js/app/src/app/pages/querybuilder/parameters.jsx @@ -0,0 +1,130 @@ +export default { + yql: { name: 'yql', type: 'String' }, + hits: { name: 'hits', type: 'Integer' }, + offset: { name: 'offset', type: 'Integer' }, + queryProfile: { name: 'queryProfile', type: 'String' }, + noCache: { name: 'noCache', type: 'Boolean' }, + groupingSessionCache: { name: 'groupingSessionCache', type: 'Boolean' }, + searchChain: { name: 'searchChain', type: 'String' }, + timeout: { name: 'timeout', type: 'Float' }, + trace: { + name: 'trace', + type: 'Parent', + children: { + timestamps: { name: 'timestamps', type: 'Boolean' }, + }, + }, + tracelevel: { + name: 'tracelevel', + type: 'Parent', + children: { + rules: { name: 'rules', type: 'Integer' }, + }, + }, + traceLevel: { name: 'traceLevel', type: 'Integer' }, + explainLevel: { name: 'explainLevel', type: 'Integer' }, + explainlevel: { name: 'explainlevel', type: 'Integer' }, + model: { + name: 'model', + type: 'Parent', + children: { + defaultIndex: { name: 'defaultIndex', type: 'String' }, + encoding: { name: 'encoding', type: 'String' }, + language: { name: 'language', type: 'String' }, + queryString: { name: 'queryString', type: 'String' }, + restrict: { name: 'restrict', type: 'List' }, + searchPath: { name: 'searchPath', type: 'String' }, + sources: { name: 'sources', type: 'List' }, + type: { name: 'type', type: 'String' }, + }, + }, + ranking: { + name: 'ranking', + type: 'Parent', + children: { + location: { name: 'location', type: 'String' }, + features: { name: 'features', type: 'String' }, + listFeatures: { name: 'listFeatures', type: 'Boolean' }, + profile: { name: 'profile', type: 'String' }, + properties: { name: 'properties', type: 'String' }, + sorting: { name: 'sorting', type: 'String' }, + freshness: { name: 'freshness', type: 'String' }, + queryCache: { name: 'queryCache', type: 'Boolean' }, + matchPhase: { + name: 'matchPhase', + type: 'Parent', + children: { + maxHits: { name: 'maxHits', type: 'Long' }, + attribute: { name: 'attribute', type: 'String' }, + ascending: { name: 'ascending', type: 'Boolean' }, + diversity: { + name: 'diversity', + type: 'Parent', + children: { + attribute: { name: 'attribute', type: 'String' }, + minGroups: { name: 'minGroups', type: 'Long' }, + }, + }, + }, + }, + }, + }, + collapse: { + name: 'collapse', + type: 'Parent', + children: { + summary: { name: 'summary', type: 'String' }, + }, + }, + collapsesize: { name: 'collapsesize', type: 'Integer' }, + collapsefield: { name: 'collapsefield', type: 'String' }, + presentation: { + name: 'presentation', + type: 'Parent', + children: { + bolding: { name: 'bolding', type: 'Boolean' }, + format: { name: 'format', type: 'String' }, + summary: { name: 'summary', type: 'String' }, + template: { name: 'template', type: 'String' }, + timing: { name: 'timing', type: 'Boolean' }, + }, + }, + pos: { + name: 'pos', + type: 'Parent', + children: { + ll: { name: 'll', type: 'String' }, + radius: { name: 'radius', type: 'String' }, + bb: { name: 'bb', type: 'List' }, + attribute: { name: 'attribute', type: 'String' }, + }, + }, + streaming: { + name: 'streaming', + type: 'Parent', + children: { + userid: { name: 'userid', type: 'Integer' }, + groupname: { name: 'groupname', type: 'String' }, + selection: { name: 'selection', type: 'String' }, + priority: { name: 'priority', type: 'String' }, + maxbucketspervisitor: { name: 'maxbucketspervisitor', type: 'Integer' }, + }, + }, + rules: { + name: 'rules', + type: 'Parent', + children: { + off: { name: 'off', type: 'Boolean' }, + rulebase: { name: 'rulebase', type: 'String' }, + }, + }, + recall: { name: 'recall', type: 'List' }, + user: { name: 'user', type: 'String' }, + metrics: { + name: 'metrics', + type: 'Parent', + children: { + ignore: { name: 'ignore', type: 'Boolean' }, + }, + }, +}; diff --git a/client/js/app/src/app/pages/querybuilder/query-builder.jsx b/client/js/app/src/app/pages/querybuilder/query-builder.jsx index 42a4ed829c3..b101e8639fb 100644 --- a/client/js/app/src/app/pages/querybuilder/query-builder.jsx +++ b/client/js/app/src/app/pages/querybuilder/query-builder.jsx @@ -1,53 +1,58 @@ import React from 'react'; import QueryInput from './Components/Text/QueryInput'; -import TextBox from './Components/Text/TextBox'; -import AddQueryInput from './Components/Buttons/AddQueryInputButton'; -import { QueryInputProvider } from './Components/Contexts/QueryInputContext'; import SendQuery from './Components/Text/SendQuery'; -import { ResponseProvider } from './Components/Contexts/ResponseContext'; -import ResponseBox from './Components/Text/ResponseBox'; -import ShowQueryButton from './Components/Buttons/ShowQueryButton'; -import { QueryProvider } from './Components/Contexts/QueryContext'; import PasteJSONButton from './Components/Buttons/PasteJSONButton'; import CopyResponseButton from './Components/Buttons/CopyResponseButton'; import DownloadJSONButton from './Components/Buttons/DownloadJSONButton'; +import { + QueryBuilderProvider, + useQueryBuilderContext, +} from 'app/pages/querybuilder/Components/Contexts/QueryBuilderProvider'; import '../../styles/agency.css'; import '../../styles/vespa.css'; -export function QueryBuilder() { +function QueryBox() { + const query = useQueryBuilderContext((ctx) => ctx.query.input); + return <textarea readOnly cols="70" rows="15" value={query}></textarea>; +} + +function ResponseBox() { + const response = useQueryBuilderContext((ctx) => ctx.http.response); return ( <> - <header> - <div className="intro container"> - <TextBox className={'intro-lead-in'}>Vespa Search Engine</TextBox> - <TextBox className={'intro-long'}> - Select the method for sending a request and construct a query. - </TextBox> - <ResponseProvider> - <QueryProvider> - <QueryInputProvider> - <SendQuery /> - <br /> - <div id="request"> - <QueryInput /> - </div> - <br /> - <AddQueryInput /> - <br /> - <PasteJSONButton /> - </QueryInputProvider> - <ShowQueryButton /> - </QueryProvider> - <TextBox className="response">Response</TextBox> - <ResponseBox /> - <CopyResponseButton /> - <DownloadJSONButton>Download response as JSON</DownloadJSONButton> - </ResponseProvider> + <textarea readOnly cols="70" rows="25" value={response} /> + <CopyResponseButton /> + <DownloadJSONButton response={response}> + Download in Jeager format + </DownloadJSONButton> + </> + ); +} + +export function QueryBuilder() { + return ( + <header> + <div className="intro container"> + <p className="intro-lead-in">Vespa Search Engine</p> + <p className="intro-long"> + Select the method for sending a request and construct a query. + </p> + <QueryBuilderProvider> + <SendQuery /> <br /> + <div id="request"> + <QueryInput /> + </div> <br /> - </div> - </header> - </> + <PasteJSONButton /> + <QueryBox /> + <p className="response">Response</p> + <ResponseBox /> + </QueryBuilderProvider> + <br /> + <br /> + </div> + </header> ); } diff --git a/client/js/app/src/app/pages/querytracer/query-tracer.jsx b/client/js/app/src/app/pages/querytracer/query-tracer.jsx index c700b73ebba..c3212c70c8a 100644 --- a/client/js/app/src/app/pages/querytracer/query-tracer.jsx +++ b/client/js/app/src/app/pages/querytracer/query-tracer.jsx @@ -1,14 +1,9 @@ -import React, { useContext } from 'react'; +import React, { useState } from 'react'; import DownloadJSONButton from '../querybuilder/Components/Buttons/DownloadJSONButton'; -import { ResponseContext } from '../querybuilder/Components/Contexts/ResponseContext'; import { Container } from 'app/components'; export function QueryTracer() { - const { response, setResponse } = useContext(ResponseContext); - - const updateResponse = (e) => { - setResponse(e.target.value); - }; + const [response, setResponse] = useState(''); return ( <Container> @@ -16,10 +11,10 @@ export function QueryTracer() { cols="70" rows="25" value={response} - onChange={updateResponse} + onChange={({ target }) => setResponse(target.value)} ></textarea> - <DownloadJSONButton> - Convert to Jeager format and download trace + <DownloadJSONButton response={response}> + Download in Jeager format </DownloadJSONButton> </Container> ); diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 2261621f31b..6ef7814c807 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -1798,6 +1798,11 @@ lodash.merge@^4.6.2: resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== +lodash@^4: + version "4.17.21" + resolved "https://registry.npm.ouryahoo.com:4443/npm-registry/api/npm/npm-registry/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== + loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -2168,9 +2173,9 @@ react-refresh@^0.13.0: resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.13.0.tgz#cbd01a4482a177a5da8d44c9755ebb1f26d5a1c1" integrity sha512-XP8A9BT0CpRBD+NYLLeIhld/RqG9+gktUjW1FkE+Vm7OCinbG1SshcK5tb9ls4kzvjZr9mOQc7HYgBngEyPAXg== -react-router-dom@^6.3.0: +react-router-dom@^6: version "6.3.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.3.0.tgz#a0216da813454e521905b5fa55e0e5176123f43d" + resolved "https://registry.npm.ouryahoo.com:4443/npm-registry/api/npm/npm-registry/react-router-dom/-/react-router-dom-6.3.0.tgz#a0216da813454e521905b5fa55e0e5176123f43d" integrity sha512-uaJj7LKytRxZNQV8+RbzJWnJ8K2nPsOOEuX7aQstlMZKQT0164C+X2w6bnkqU3sjtLvpd5ojrezAyfZ1+0sStw== dependencies: history "^5.2.0" @@ -2464,6 +2469,11 @@ use-composed-ref@^1.3.0: resolved "https://registry.yarnpkg.com/use-composed-ref/-/use-composed-ref-1.3.0.tgz#3d8104db34b7b264030a9d916c5e94fbe280dbda" integrity sha512-GLMG0Jc/jiKov/3Ulid1wbv3r54K9HlMW29IWcDFPEqFkSO2nS0MuefWgMJpeHQ9YJeXDL3ZUF+P3jdXlZX/cQ== +use-context-selector@^1: + version "1.4.1" + resolved "https://registry.npm.ouryahoo.com:4443/npm-registry/api/npm/npm-registry/use-context-selector/-/use-context-selector-1.4.1.tgz#eb96279965846b72915d7f899b8e6ef1d768b0ae" + integrity sha512-Io2ArvcRO+6MWIhkdfMFt+WKQX+Vb++W8DS2l03z/Vw/rz3BclKpM0ynr4LYGyU85Eke+Yx5oIhTY++QR0ZDoA== + use-isomorphic-layout-effect@^1.1.1: version "1.1.2" resolved "https://registry.yarnpkg.com/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.1.2.tgz#497cefb13d863d687b08477d9e5a164ad8c1a6fb" diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java index f2a4a9736c3..695fecb6314 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java @@ -1,20 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.distribution.Distribution; -import com.yahoo.vdslib.distribution.Group; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; -import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; -import com.yahoo.vespa.clustercontroller.core.status.statuspage.HtmlTable; -import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRenderer; import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest; - import java.util.Collection; import java.util.Collections; import java.util.List; @@ -46,61 +40,6 @@ public class ContentCluster { setNodes(configuredNodes, new NodeListener() {}); } - // TODO move out, this doesn't belong in a domain model class - public void writeHtmlState( - final VdsClusterHtmlRenderer vdsClusterHtmlRenderer, - final StringBuilder sb, - final Timer timer, - final ClusterStateBundle state, - final ClusterStatsAggregator statsAggregator, - final Distribution distribution, - final FleetControllerOptions options, - final EventLog eventLog) { - - final VdsClusterHtmlRenderer.Table table = - vdsClusterHtmlRenderer.createNewClusterHtmlTable(clusterName, slobrokGenerationCount); - - if (state.clusterFeedIsBlocked()) { // Implies FeedBlock != null - table.appendRaw("<h3 style=\"color: red\">Cluster feeding is blocked!</h3>\n"); - table.appendRaw(String.format("<p>Summary: <strong>%s</strong></p>\n", - HtmlTable.escape(state.getFeedBlockOrNull().getDescription()))); - } - - final List<Group> groups = LeafGroups.enumerateFrom(distribution.getRootGroup()); - - for (int j=0; j<groups.size(); ++j) { - final Group group = groups.get(j); - assert(group != null); - final String localName = group.getUnixStylePath(); - assert(localName != null); - final TreeMap<Integer, NodeInfo> storageNodeInfoByIndex = new TreeMap<>(); - final TreeMap<Integer, NodeInfo> distributorNodeInfoByIndex = new TreeMap<>(); - for (ConfiguredNode configuredNode : group.getNodes()) { - storeNodeInfo(configuredNode.index(), NodeType.STORAGE, storageNodeInfoByIndex); - storeNodeInfo(configuredNode.index(), NodeType.DISTRIBUTOR, distributorNodeInfoByIndex); - } - table.renderNodes( - storageNodeInfoByIndex, - distributorNodeInfoByIndex, - timer, - state, - statsAggregator, - options.minMergeCompletionRatio, - options.maxPrematureCrashes, - options.clusterFeedBlockLimit, - eventLog, - clusterName, - localName); - } - table.addTable(sb, options.stableStateTimePeriod); - } - - private void storeNodeInfo(int nodeIndex, NodeType nodeType, Map<Integer, NodeInfo> nodeInfoByIndex) { - NodeInfo nodeInfo = getNodeInfo(new Node(nodeType, nodeIndex)); - if (nodeInfo == null) return; - nodeInfoByIndex.put(nodeIndex, nodeInfo); - } - public Distribution getDistribution() { return distribution; } public void setDistribution(Distribution distribution) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 7f385c6077c..8b335e877cd 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -12,8 +12,8 @@ import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; -import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; +import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; import com.yahoo.vespa.clustercontroller.core.listeners.SystemStateListener; import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator; import com.yahoo.vespa.clustercontroller.core.rpc.RpcServer; @@ -27,7 +27,6 @@ import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageRespon import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServerInterface; import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter; - import java.io.FileNotFoundException; import java.util.ArrayDeque; import java.util.ArrayList; @@ -45,7 +44,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.stream.Stream; public class FleetController implements NodeListener, SlobrokListener, SystemStateListener, Runnable, RemoteClusterControllerTaskScheduler { @@ -155,10 +153,8 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta new ClusterStateRequestHandler(stateVersionTracker)); this.statusRequestRouter.addHandler( "^/$", - new LegacyIndexPageRequestHandler( - timer, options.showLocalSystemStatesInEventLog, cluster, - masterElectionHandler, stateVersionTracker, - eventLog, timer.getCurrentTimeInMillis(), dataExtractor)); + new LegacyIndexPageRequestHandler(timer, cluster, masterElectionHandler, stateVersionTracker, eventLog, + timer.getCurrentTimeInMillis(), dataExtractor)); propagateOptions(); } @@ -505,9 +501,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta cluster.setSlobrokGenerationCount(0); } - configuredBucketSpaces = Collections.unmodifiableSet( - Stream.of(FixedBucketSpaces.defaultSpace(), FixedBucketSpaces.globalSpace()) - .collect(Collectors.toSet())); + configuredBucketSpaces = Set.of(FixedBucketSpaces.defaultSpace(), FixedBucketSpaces.globalSpace()); stateVersionTracker.setMinMergeCompletionRatio(options.minMergeCompletionRatio); communicator.propagateOptions(options); @@ -634,7 +628,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta didWork |= metricUpdater.forWork("processAnyPendingStatusPageRequest", this::processAnyPendingStatusPageRequest); if ( ! isRunning()) { return; } if (rpcServer != null) { - didWork |= metricUpdater.forWork("handleRpcRequests", () -> rpcServer.handleRpcRequests(cluster, consolidatedClusterState(), this, this)); + didWork |= metricUpdater.forWork("handleRpcRequests", () -> rpcServer.handleRpcRequests(cluster, consolidatedClusterState(), this)); } if ( ! isRunning()) { return; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java index a5a29b8d7f1..502fc37dead 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java @@ -11,10 +11,9 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.Values; -import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.ClusterState; +import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; -import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.core.ActivateClusterStateVersionRequest; import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle; import com.yahoo.vespa.clustercontroller.core.Communicator; @@ -23,7 +22,7 @@ import com.yahoo.vespa.clustercontroller.core.GetNodeStateRequest; import com.yahoo.vespa.clustercontroller.core.NodeInfo; import com.yahoo.vespa.clustercontroller.core.SetClusterStateRequest; import com.yahoo.vespa.clustercontroller.core.Timer; - +import java.util.logging.Level; import java.util.logging.Logger; import static com.google.common.base.Preconditions.checkArgument; @@ -71,7 +70,7 @@ public class RPCCommunicator implements Communicator { checkArgument(nodeStateRequestTimeoutIntervalStartPercentage >= 0); checkArgument(nodeStateRequestTimeoutIntervalStartPercentage <= 100); checkArgument(nodeStateRequestTimeoutIntervalStopPercentage >= nodeStateRequestTimeoutIntervalStartPercentage); - checkArgument(nodeStateRequestTimeoutIntervalStartPercentage <= 100); + checkArgument(nodeStateRequestTimeoutIntervalStopPercentage <= 100); checkArgument(nodeStateRequestRoundTripTimeMaxSeconds >= 0); this.nodeStateRequestTimeoutIntervalMaxSeconds = nodeStateRequestTimeoutIntervalMaxMs / 1000D; this.nodeStateRequestTimeoutIntervalStartPercentage = nodeStateRequestTimeoutIntervalStartPercentage; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java index 6e416ce4906..dc21693dcdb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java @@ -15,8 +15,6 @@ import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.BackOffPolicy; import com.yahoo.jrt.slobrok.api.Register; import com.yahoo.jrt.slobrok.api.SlobrokList; -import java.util.logging.Level; - import com.yahoo.net.HostName; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; @@ -27,34 +25,34 @@ import com.yahoo.vespa.clustercontroller.core.ContentCluster; import com.yahoo.vespa.clustercontroller.core.MasterElectionHandler; import com.yahoo.vespa.clustercontroller.core.NodeInfo; import com.yahoo.vespa.clustercontroller.core.Timer; -import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; - import java.io.PrintWriter; import java.io.StringWriter; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; public class RpcServer { - private static Logger log = Logger.getLogger(RpcServer.class.getName()); + private static final Logger log = Logger.getLogger(RpcServer.class.getName()); private final Timer timer; private final Object monitor; private final String clusterName; private final int fleetControllerIndex; - private String slobrokConnectionSpecs[]; + private String[] slobrokConnectionSpecs; private int port = 0; private Supervisor supervisor; private Acceptor acceptor; private Register register; private final List<Request> rpcRequests = new LinkedList<>(); private MasterElectionHandler masterHandler; - private BackOffPolicy slobrokBackOffPolicy; + private final BackOffPolicy slobrokBackOffPolicy; private long lastConnectErrorTime = 0; private String lastConnectError = ""; @@ -81,10 +79,10 @@ public class RpcServer { return "storage/cluster." + clusterName + "/fleetcontroller/" + fleetControllerIndex; } - public void setSlobrokConnectionSpecs(String slobrokConnectionSpecs[], int port) throws ListenFailedException, UnknownHostException { - if (this.slobrokConnectionSpecs == null || !this.slobrokConnectionSpecs.equals(slobrokConnectionSpecs) // TODO: <-- probably a bug - || this.port != port) - { + public void setSlobrokConnectionSpecs(String[] slobrokConnectionSpecs, int port) throws ListenFailedException, UnknownHostException { + if (this.slobrokConnectionSpecs == null + || !Arrays.equals(this.slobrokConnectionSpecs, slobrokConnectionSpecs) + || this.port != port) { this.slobrokConnectionSpecs = slobrokConnectionSpecs; this.port = port; disconnect(); @@ -105,7 +103,7 @@ public class RpcServer { log.log(Level.FINE, () -> "Fleetcontroller " + fleetControllerIndex + ": Attempting to bind to port " + port); acceptor = supervisor.listen(new Spec(port)); log.log(Level.FINE, () -> "Fleetcontroller " + fleetControllerIndex + ": RPC server listening to port " + acceptor.port()); - StringBuffer slobroks = new StringBuffer("("); + StringBuilder slobroks = new StringBuilder("("); for (String s : slobrokConnectionSpecs) { slobroks.append(" ").append(s); } @@ -185,10 +183,7 @@ public class RpcServer { } } - public boolean handleRpcRequests(ContentCluster cluster, ClusterState systemState, - NodeListener changeListener, - SlobrokListener addedListener) - { + public boolean handleRpcRequests(ContentCluster cluster, ClusterState systemState, NodeListener changeListener) { boolean handledAnyRequests = false; if (!isConnected()) { long time = timer.getCurrentTimeInMillis(); @@ -255,8 +250,6 @@ public class RpcServer { NodeType nodeType = NodeType.get(req.parameters().get(0).asString()); int nodeIndex = req.parameters().get(1).asInt32(); Node node = new Node(nodeType, nodeIndex); - // First parameter is current state in system state - NodeState ns = systemState.getNodeState(node); req.returnValues().add(new StringValue(systemState.getNodeState(node).serialize())); // Second parameter is state node is reporting NodeInfo nodeInfo = cluster.getNodeInfo(node); @@ -276,7 +269,7 @@ public class RpcServer { throw new IllegalStateException("Invalid slobrok address '" + slobrokAddress + "'."); } NodeType nodeType = NodeType.get(slobrokAddress.substring(nextButLastSlash + 1, lastSlash)); - Integer nodeIndex = Integer.valueOf(slobrokAddress.substring(lastSlash + 1)); + int nodeIndex = Integer.parseInt(slobrokAddress.substring(lastSlash + 1)); NodeInfo node = cluster.getNodeInfo(new Node(nodeType, nodeIndex)); if (node == null) throw new IllegalStateException("Cannot set wanted state of node " + new Node(nodeType, nodeIndex) + ". Index does not correspond to a configured node."); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java index 378f65f7235..96dc114c734 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java @@ -1,14 +1,29 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.status; -import com.yahoo.vdslib.state.ClusterState; -import com.yahoo.vespa.clustercontroller.core.*; +import com.yahoo.vdslib.distribution.ConfiguredNode; +import com.yahoo.vdslib.distribution.Group; +import com.yahoo.vdslib.state.Node; +import com.yahoo.vdslib.state.NodeType; +import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle; +import com.yahoo.vespa.clustercontroller.core.ClusterStateHistoryEntry; +import com.yahoo.vespa.clustercontroller.core.ContentCluster; +import com.yahoo.vespa.clustercontroller.core.EventLog; +import com.yahoo.vespa.clustercontroller.core.FleetControllerOptions; +import com.yahoo.vespa.clustercontroller.core.LeafGroups; +import com.yahoo.vespa.clustercontroller.core.MasterElectionHandler; +import com.yahoo.vespa.clustercontroller.core.NodeInfo; +import com.yahoo.vespa.clustercontroller.core.RealTimer; +import com.yahoo.vespa.clustercontroller.core.StateVersionTracker; import com.yahoo.vespa.clustercontroller.core.Timer; +import com.yahoo.vespa.clustercontroller.core.status.statuspage.HtmlTable; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageResponse; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer; import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRenderer; - -import java.util.*; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import java.util.TreeMap; /** * @author Haakon Humberset @@ -22,15 +37,15 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa private final EventLog eventLog; private final long startedTime; private final RunDataExtractor data; - private final boolean showLocalSystemStatesInLog; - public LegacyIndexPageRequestHandler(Timer timer, boolean showLocalSystemStatesInLog, ContentCluster cluster, + public LegacyIndexPageRequestHandler(Timer timer, + ContentCluster cluster, MasterElectionHandler masterElectionHandler, StateVersionTracker stateVersionTracker, - EventLog eventLog, long startedTime, RunDataExtractor data) - { + EventLog eventLog, + long startedTime, + RunDataExtractor data) { this.timer = timer; - this.showLocalSystemStatesInLog = showLocalSystemStatesInLog; this.cluster = cluster; this.masterElectionHandler = masterElectionHandler; this.stateVersionTracker = stateVersionTracker; @@ -59,18 +74,9 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa content.append("<tr><td>Cluster controller uptime:</td><td align=\"right\">" + RealTimer.printDuration(currentTime - startedTime) + "</td></tr></table>"); if (masterElectionHandler.isAmongNthFirst(data.getOptions().stateGatherCount)) { // Table overview of all the nodes - cluster.writeHtmlState( - new VdsClusterHtmlRenderer(), - content, - timer, - stateVersionTracker.getVersionedClusterStateBundle(), - stateVersionTracker.getAggregatedClusterStats(), - data.getOptions().storageDistribution, - data.getOptions(), - eventLog - ); + writeHtmlState(cluster, content, timer, stateVersionTracker, data.getOptions(), eventLog); // Current cluster state and cluster state history - writeHtmlState(stateVersionTracker, content, request); + writeHtmlState(stateVersionTracker, content); } else { // Overview of current config data.getOptions().writeHtmlState(content); @@ -87,14 +93,7 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa return response; } - public void writeHtmlState(StateVersionTracker stateVersionTracker, StringBuilder sb, StatusPageServer.HttpRequest request) { - boolean showLocal = showLocalSystemStatesInLog; - if (request.hasQueryParameter("showlocal")) { - showLocal = true; - } else if (request.hasQueryParameter("hidelocal")) { - showLocal = false; - } - + public void writeHtmlState(StateVersionTracker stateVersionTracker, StringBuilder sb) { sb.append("<h2 id=\"clusterstates\">Cluster states</h2>\n"); writeClusterStates(sb, stateVersionTracker.getVersionedClusterStateBundle()); @@ -153,4 +152,53 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa sb.append("</td></tr>\n"); } + private void writeHtmlState(ContentCluster cluster, + StringBuilder sb, + Timer timer, + StateVersionTracker stateVersionTracker, + FleetControllerOptions options, + EventLog eventLog) { + VdsClusterHtmlRenderer renderer = new VdsClusterHtmlRenderer(); + VdsClusterHtmlRenderer.Table table = renderer.createNewClusterHtmlTable(cluster.getName(), cluster.getSlobrokGenerationCount()); + + ClusterStateBundle state = stateVersionTracker.getVersionedClusterStateBundle(); + if (state.clusterFeedIsBlocked()) { // Implies FeedBlock != null + table.appendRaw("<h3 style=\"color: red\">Cluster feeding is blocked!</h3>\n"); + table.appendRaw(String.format("<p>Summary: <strong>%s</strong></p>\n", + HtmlTable.escape(state.getFeedBlockOrNull().getDescription()))); + } + + List<Group> groups = LeafGroups.enumerateFrom(options.storageDistribution.getRootGroup()); + + for (Group group : groups) { + assert (group != null); + String localName = group.getUnixStylePath(); + assert (localName != null); + TreeMap<Integer, NodeInfo> storageNodeInfoByIndex = new TreeMap<>(); + TreeMap<Integer, NodeInfo> distributorNodeInfoByIndex = new TreeMap<>(); + for (ConfiguredNode configuredNode : group.getNodes()) { + storeNodeInfo(cluster, configuredNode.index(), NodeType.STORAGE, storageNodeInfoByIndex); + storeNodeInfo(cluster, configuredNode.index(), NodeType.DISTRIBUTOR, distributorNodeInfoByIndex); + } + table.renderNodes(storageNodeInfoByIndex, + distributorNodeInfoByIndex, + timer, + state, + stateVersionTracker.getAggregatedClusterStats(), + options.minMergeCompletionRatio, + options.maxPrematureCrashes, + options.clusterFeedBlockLimit, + eventLog, + cluster.getName(), + localName); + } + table.addTable(sb, options.stableStateTimePeriod); + } + + private void storeNodeInfo(ContentCluster cluster, int nodeIndex, NodeType nodeType, Map<Integer, NodeInfo> nodeInfoByIndex) { + NodeInfo nodeInfo = cluster.getNodeInfo(new Node(nodeType, nodeIndex)); + if (nodeInfo == null) return; + nodeInfoByIndex.put(nodeIndex, nodeInfo); + } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java index c167c82aa90..af067cc394f 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java @@ -21,7 +21,6 @@ import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator; import com.yahoo.vespa.clustercontroller.core.rpc.RPCUtil; - import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -39,7 +38,7 @@ import java.util.stream.Collectors; */ public class DummyVdsNode { - public static Logger log = Logger.getLogger(DummyVdsNode.class.getName()); + private static final Logger log = Logger.getLogger(DummyVdsNode.class.getName()); private final String[] slobrokConnectionSpecs; private final String clusterName; @@ -55,7 +54,7 @@ public class DummyVdsNode { private final Timer timer; private boolean failSetSystemStateRequests = false; private boolean resetTimestampOnReconnect = false; - private final Map<Node, Long> highestStartTimestamps = new TreeMap<Node, Long>(); + private final Map<Node, Long> highestStartTimestamps = new TreeMap<>(); int timedOutStateReplies = 0; int outdatedStateReplies = 0; int immediateStateReplies = 0; @@ -88,7 +87,7 @@ public class DummyVdsNode { private final Thread messageResponder = new Thread() { public void run() { - log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this.toString() + ": starting message reponder thread"); + log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": starting message responder thread"); while (true) { synchronized (timer) { if (isInterrupted()) break; @@ -96,7 +95,7 @@ public class DummyVdsNode { for (Iterator<Req> it = waitingRequests.iterator(); it.hasNext(); ) { Req r = it.next(); if (r.timeout <= currentTime) { - log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this.toString() + ": Responding to node state request at time " + currentTime); + log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": Responding to node state request at time " + currentTime); r.request.returnValues().add(new StringValue(nodeState.serialize())); if (r.request.methodName().equals("getnodestate3")) { r.request.returnValues().add(new StringValue("No host info in dummy implementation")); @@ -113,7 +112,7 @@ public class DummyVdsNode { } } } - log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this.toString() + ": shut down message reponder thread"); + log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": shut down message responder thread"); } }; @@ -170,31 +169,16 @@ public class DummyVdsNode { registeredInSlobrok = false; } - void disconnect() { disconnectImmediately(); } - void disconnectImmediately() { disconnect(false, 0, false); } - void disconnectBreakConnection() { disconnect(true, FleetControllerTest.timeoutMS, false); } - void disconnectAsShutdown() { disconnect(true, FleetControllerTest.timeoutMS, true); } - private void disconnect(boolean waitForPendingNodeStateRequest, long timeoutms, boolean setStoppingStateFirst) { - log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this.toString() + ": Breaking connection." + (waitForPendingNodeStateRequest ? " Waiting for pending state first." : "")); - if (waitForPendingNodeStateRequest) { - this.waitForPendingGetNodeStateRequest(timeoutms); - } - if (setStoppingStateFirst) { - NodeState newState = nodeState.clone(); - newState.setState(State.STOPPING); - // newState.setDescription("Received signal 15 (SIGTERM - Termination signal)"); - // Altered in storageserver implementation. Updating now to fit - newState.setDescription("controlled shutdown"); - setNodeState(newState); - // Sleep a bit in hopes of answer being written before shutting down socket - try{ Thread.sleep(10); } catch (InterruptedException e) { /* ignore */ } - } + void disconnectImmediately() { disconnect(); } + + void disconnect() { + log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": Breaking connection."); if (supervisor == null) return; register.shutdown(); acceptor.shutdown().join(); supervisor.transport().shutdown().join(); supervisor = null; - log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this.toString() + ": Done breaking connection."); + log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": Done breaking connection."); } public String toString() { @@ -210,11 +194,11 @@ public class DummyVdsNode { public int getStateCommunicationVersion() { return stateCommunicationVersion; } - void waitForSystemStateVersion(int version, long timeout) { + void waitForSystemStateVersion(int version) { try { long startTime = System.currentTimeMillis(); while (getLatestSystemStateVersion().orElse(-1) < version) { - if ( (System.currentTimeMillis() - startTime) > timeout) + if ( (System.currentTimeMillis() - startTime) > (long) FleetControllerTest.timeoutMS) throw new RuntimeException("Timed out waiting for state version " + version + " in " + this); Thread.sleep(10); } @@ -237,33 +221,6 @@ public class DummyVdsNode { } } - private void waitForPendingGetNodeStateRequest(long timeout) { - long startTime = System.currentTimeMillis(); - long endTime = startTime + timeout; - log.log(Level.FINE, () -> "Dummy node " + this + " waiting for pending node state request."); - while (true) { - synchronized(timer) { - if (!waitingRequests.isEmpty()) { - log.log(Level.FINE, () -> "Dummy node " + this + " has pending request, returning."); - return; - } - try { - log.log(Level.FINE, "Dummy node " + this + " waiting " + (endTime - startTime) + " ms for pending request."); - timer.wait(endTime - startTime); - } catch (InterruptedException e) { /* ignore */ } - log.log(Level.FINE, () -> "Dummy node " + this + " woke up to recheck."); - } - startTime = System.currentTimeMillis(); - if (startTime >= endTime) { - log.log(Level.FINE, () -> "Dummy node " + this + " timeout passed. Don't have pending request."); - if (!waitingRequests.isEmpty()) { - log.log(Level.FINE, () -> "Dummy node " + this + ". Non-empty set of waiting requests"); - } - throw new IllegalStateException("Timeout. No pending get node state request pending after waiting " + timeout + " milliseconds."); - } - } - } - void replyToPendingNodeStateRequests() { for(Req req : waitingRequests) { log.log(Level.FINE, () -> "Dummy node " + this + " answering pending node state request."); @@ -422,7 +379,7 @@ public class DummyVdsNode { for (Iterator<Req> it = waitingRequests.iterator(); it.hasNext(); ) { Req r = it.next(); if (r.request.parameters().size() > 2 && r.request.parameters().get(2).asInt32() == index) { - log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this.toString() + ": Responding to node state reply from controller " + index + " as we received new one"); + log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": Responding to node state reply from controller " + index + " as we received new one"); r.request.returnValues().add(new StringValue(nodeState.serialize())); r.request.returnValues().add(new StringValue("No host info from dummy implementation")); r.request.returnRequest(); @@ -448,9 +405,9 @@ public class DummyVdsNode { NodeState givenState = (oldState.equals("unknown") ? null : NodeState.deserialize(type, oldState)); if (givenState != null && (givenState.equals(nodeState) || sentReply)) { log.log(Level.FINE, () -> "Dummy node " + this + ": Has same state as reported " + givenState + ". Queing request. Timeout is " + timeout + " ms. " - + "Will be answered at time " + (timer.getCurrentTimeInMillis() + timeout * 800l / 1000)); + + "Will be answered at time " + (timer.getCurrentTimeInMillis() + timeout * 800L / 1000)); req.detach(); - waitingRequests.add(new Req(req, timer.getCurrentTimeInMillis() + timeout * 800l / 1000)); + waitingRequests.add(new Req(req, timer.getCurrentTimeInMillis() + timeout * 800L / 1000)); log.log(Level.FINE, () -> "Dummy node " + this + " has now " + waitingRequests.size() + " entries and is " + (waitingRequests.isEmpty() ? "empty" : "not empty")); timer.notifyAll(); } else { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java index 7c61423ac2b..24e65a89d2b 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java @@ -1128,7 +1128,7 @@ public class StateChangeTest extends FleetControllerTest { // At this time, node taken down should have cluster states with all starting timestamps set. Others node should not. for (DummyVdsNode node : nodes) { - node.waitForSystemStateVersion(waiter.getCurrentSystemState().getVersion(), timeoutMS); + node.waitForSystemStateVersion(waiter.getCurrentSystemState().getVersion()); List<ClusterState> states = node.getSystemStatesReceived(); ClusterState lastState = states.get(0); StringBuilder stateHistory = new StringBuilder(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index 53e2592e0a6..7539f7b4cf2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -274,6 +274,11 @@ public class ZmsClientMock implements ZmsClient { } @Override + public void deleteSubdomain(AthenzDomain parent, String name) { + athenz.domains.remove(new AthenzDomain(parent.getName() + "." + name)); + } + + @Override public void close() {} private static AthenzDomain getTenantDomain(AthenzResourceName resource) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java index 541eb3dbe90..1ef1bc5106c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java @@ -35,4 +35,9 @@ public class NoopRoleService implements RoleService { @Override public void maintainRoles(List<TenantName> tenants) { } + + @Override + public void cleanupRoles(List<TenantName> tenants) { + + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/RoleService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/RoleService.java index bc661077537..0a35893a7c4 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/RoleService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/RoleService.java @@ -27,4 +27,6 @@ public interface RoleService { * Maintain roles for the tenants in the system. Create missing roles, update trust. */ void maintainRoles(List<TenantName> tenants); + + void cleanupRoles(List<TenantName> deletedTenants); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index a259ed2fdef..ab2e0312b15 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -33,6 +33,7 @@ import static java.time.temporal.ChronoUnit.SECONDS; public class ControllerMaintenance extends AbstractComponent { private final Upgrader upgrader; + private final OsUpgradeScheduler osUpgradeScheduler; private final List<Maintainer> maintainers = new CopyOnWriteArrayList<>(); @Inject @@ -40,7 +41,9 @@ public class ControllerMaintenance extends AbstractComponent { public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement, AthenzClientFactory athenzClientFactory) { Intervals intervals = new Intervals(controller.system()); upgrader = new Upgrader(controller, intervals.defaultInterval); + osUpgradeScheduler = new OsUpgradeScheduler(controller, intervals.osUpgradeScheduler); maintainers.add(upgrader); + maintainers.add(osUpgradeScheduler); maintainers.addAll(osUpgraders(controller, intervals.osUpgrader)); maintainers.add(new DeploymentExpirer(controller, intervals.defaultInterval)); maintainers.add(new DeploymentUpgrader(controller, intervals.defaultInterval)); @@ -54,7 +57,6 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new SystemUpgrader(controller, intervals.systemUpgrader)); maintainers.add(new JobRunner(controller, intervals.jobRunner)); maintainers.add(new OsVersionStatusUpdater(controller, intervals.osVersionStatusUpdater)); - maintainers.add(new OsUpgradeScheduler(controller, intervals.osUpgradeScheduler)); maintainers.add(new ContactInformationMaintainer(controller, intervals.contactInformationMaintainer)); maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher)); maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer())); @@ -80,6 +82,8 @@ public class ControllerMaintenance extends AbstractComponent { public Upgrader upgrader() { return upgrader; } + public OsUpgradeScheduler osUpgradeScheduler() { return osUpgradeScheduler; } + @Override public void deconstruct() { maintainers.forEach(Maintainer::shutdown); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 1454d78ce33..b051590ac5a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -104,7 +104,7 @@ public abstract class InfrastructureUpgrader<TARGET extends VersionTarget> exten Set<SystemApplication> dependencies = kv.getValue(); boolean allConverged = dependencies.stream().allMatch(app -> convergedOn(target, app, zone, nodeSlice)); if (allConverged) { - if (changeTargetTo(target, application, zone, nodeSlice)) { + if (changeTargetTo(target, application, zone)) { upgrade(target, application, zone); } converged &= convergedOn(target, application, zone, nodeSlice); @@ -114,7 +114,7 @@ public abstract class InfrastructureUpgrader<TARGET extends VersionTarget> exten } /** Returns whether target version for application in zone should be changed */ - protected abstract boolean changeTargetTo(TARGET target, SystemApplication application, ZoneApi zone, NodeSlice nodeSlice); + protected abstract boolean changeTargetTo(TARGET target, SystemApplication application, ZoneApi zone); /** Upgrade component to target version. Implementation should be idempotent */ protected abstract void upgrade(TARGET target, SystemApplication application, ZoneApi zone); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index 111931b638b..644a8c6c1ed 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -14,7 +14,9 @@ import java.time.Duration; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; import java.util.Objects; import java.util.Optional; @@ -32,23 +34,39 @@ public class OsUpgradeScheduler extends ControllerMaintainer { @Override protected double maintain() { Instant now = controller().clock().instant(); - if (!canTriggerAt(now)) return 1.0; for (var cloud : controller().clouds()) { - Release release = releaseIn(cloud); - upgradeTo(release, cloud, now); + Optional<Change> change = changeIn(cloud); + if (change.isEmpty()) continue; + if (!change.get().scheduleAt(now)) continue; + controller().upgradeOsIn(cloud, change.get().version(), change.get().upgradeBudget(), false); } return 1.0; } - /** Upgrade to given release in cloud */ - private void upgradeTo(Release release, CloudName cloud, Instant now) { + /** Returns the wanted change for given cloud, if any */ + public Optional<Change> changeIn(CloudName cloud) { Optional<OsVersionTarget> currentTarget = controller().osVersionTarget(cloud); - if (currentTarget.isEmpty()) return; - if (upgradingToNewMajor(cloud)) return; // Skip further upgrades until major version upgrade is complete - - Version version = release.version(currentTarget.get(), now); - if (!version.isAfter(currentTarget.get().osVersion().version())) return; - controller().upgradeOsIn(cloud, version, release.upgradeBudget(), false); + if (currentTarget.isEmpty()) return Optional.empty(); + if (upgradingToNewMajor(cloud)) return Optional.empty(); // Skip further upgrades until major version upgrade is complete + + Release release = releaseIn(cloud); + Instant instant = controller().clock().instant(); + Version wantedVersion = release.version(currentTarget.get(), instant); + Version currentVersion = currentTarget.get().version(); + if (release instanceof CalendarVersionedRelease) { + // Estimate the next change + while (!wantedVersion.isAfter(currentVersion)) { + instant = instant.plus(Duration.ofDays(1)); + wantedVersion = release.version(currentTarget.get(), instant); + } + } else if (!wantedVersion.isAfter(currentVersion)) { + return Optional.empty(); // No change right now, and we cannot predict the next change for this kind of release + } + // Find trigger time + while (!canTriggerAt(instant)) { + instant = instant.truncatedTo(ChronoUnit.HOURS).plus(Duration.ofHours(1)); + } + return Optional.of(new Change(wantedVersion, release.upgradeBudget(), instant)); } private boolean upgradingToNewMajor(CloudName cloud) { @@ -58,23 +76,24 @@ public class OsUpgradeScheduler extends ControllerMaintainer { .count() > 1; } - private boolean canTriggerAt(Instant instant) { - int hourOfDay = instant.atZone(ZoneOffset.UTC).getHour(); - int dayOfWeek = instant.atZone(ZoneOffset.UTC).getDayOfWeek().getValue(); - // Upgrade can only be scheduled between 07:00 (02:00 in CD systems) and 12:59 UTC, Monday-Thursday - int startHour = controller().system().isCd() ? 2 : 7; - return hourOfDay >= startHour && hourOfDay <= 12 && dayOfWeek < 5; - } - private Release releaseIn(CloudName cloud) { boolean useTaggedRelease = controller().zoneRegistry().zones().all().reprovisionToUpgradeOs().in(cloud) - .zones().isEmpty(); + .zones().isEmpty(); if (useTaggedRelease) { return new TaggedRelease(controller().system(), controller().serviceRegistry().artifactRepository()); } return new CalendarVersionedRelease(controller().system()); } + private boolean canTriggerAt(Instant instant) { + ZonedDateTime dateTime = instant.atZone(ZoneOffset.UTC); + int hourOfDay = dateTime.getHour(); + int dayOfWeek = dateTime.getDayOfWeek().getValue(); + // Upgrade can only be scheduled between 07:00 (02:00 in CD systems) and 12:59 UTC, Monday-Thursday + int startHour = controller().system().isCd() ? 2 : 7; + return hourOfDay >= startHour && hourOfDay <= 12 && dayOfWeek < 5; + } + private interface Release { /** The version number of this */ @@ -85,6 +104,22 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } + /** OS version change, its budget and the earliest time it can be scheduled */ + public record Change(Version version, Duration upgradeBudget, Instant scheduleAt) { + + public Change { + Objects.requireNonNull(version); + Objects.requireNonNull(upgradeBudget); + Objects.requireNonNull(scheduleAt); + } + + /** Returns whether this can be scheduled at given instant */ + public boolean scheduleAt(Instant instant) { + return !instant.isBefore(scheduleAt); + } + + } + /** OS release based on a tag */ private record TaggedRelease(SystemName system, ArtifactRepository artifactRepository) implements Release { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 46b504cadff..f4dcf7f6088 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -76,7 +76,7 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { } @Override - protected boolean changeTargetTo(OsVersionTarget target, SystemApplication application, ZoneApi zone, NodeSlice nodeSlice) { + protected boolean changeTargetTo(OsVersionTarget target, SystemApplication application, ZoneApi zone) { if (!application.shouldUpgradeOs()) return false; return controller().serviceRegistry().configServer().nodeRepository() .targetVersionsOf(zone.getVirtualId()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 892ad669e4b..205fb7e0e79 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -100,7 +100,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { } if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots); - if (systemName.isPublic() && systemName.isCd()) reportAllScalingEvents(); + if (systemName.isPublic()) reportAllScalingEvents(); updateDeploymentCost(resourceSnapshots); return 1.0; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 86587c8e9f7..8e74ef9a983 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -74,16 +74,15 @@ public class SystemUpgrader extends InfrastructureUpgrader<VespaVersionTarget> { } @Override - protected boolean changeTargetTo(VespaVersionTarget target, SystemApplication application, ZoneApi zone, NodeSlice nodeSlice) { + protected boolean changeTargetTo(VespaVersionTarget target, SystemApplication application, ZoneApi zone) { if (application.hasApplicationPackage()) { // For applications with package we do not have a zone-wide version target. This means that we must check // the wanted version of each node. boolean zoneHasSharedRouting = controller().zoneRegistry().routingMethods(zone.getId()).stream() .anyMatch(RoutingMethod::isShared); - return versionOf(nodeSlice, zone, application, Node::wantedVersion) + return versionOf(NodeSlice.ALL, zone, application, Node::wantedVersion) .map(wantedVersion -> !wantedVersion.equals(target.version())) .orElse(zoneHasSharedRouting); // Always upgrade if zone uses shared routing, but has no nodes allocated yet - } return controller().serviceRegistry().configServer().nodeRepository() .targetVersionsOf(zone.getId()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java index dad836ca2de..820c67f2d44 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java @@ -33,21 +33,15 @@ public class TenantRoleMaintainer extends ControllerMaintainer { .map(Tenant::name) .collect(Collectors.toList()); roleService.maintainRoles(tenantsWithRoles); + + var deletedTenants = controller().tenants().asList(true).stream() + .filter(tenant -> tenant.type() == Tenant.Type.deleted) + .map(Tenant::name) + .toList(); + roleService.cleanupRoles(deletedTenants); + return 1.0; } - private boolean hasProductionDeployment(TenantName tenant) { - return controller().applications().asList(tenant).stream() - .map(Application::productionInstances) - .anyMatch(Predicate.not(Map::isEmpty)); - } - private boolean hasPerfDeployment(TenantName tenant) { - List<ZoneId> perfZones = controller().zoneRegistry().zones().controllerUpgraded().in(Environment.perf).ids(); - return controller().applications().asList(tenant).stream() - .map(Application::instances) - .flatMap(instances -> instances.values().stream()) - .flatMap(instance -> instance.deployments().values().stream()) - .anyMatch(x -> perfZones.contains(x.zone())); - } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java index 853739ee9c3..0e764b98514 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -22,6 +22,9 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; +import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; +import com.yahoo.vespa.hosted.controller.maintenance.OsUpgradeScheduler; +import com.yahoo.vespa.hosted.controller.maintenance.OsUpgradeScheduler.Change; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.yolean.Exceptions; @@ -47,22 +50,24 @@ import java.util.stream.Collectors; public class OsApiHandler extends AuditLoggingRequestHandler { private final Controller controller; + private final OsUpgradeScheduler osUpgradeScheduler; - public OsApiHandler(Context ctx, Controller controller) { + public OsApiHandler(Context ctx, Controller controller, ControllerMaintenance controllerMaintenance) { super(ctx, controller.auditLogger()); this.controller = controller; + this.osUpgradeScheduler = controllerMaintenance.osUpgradeScheduler(); } @Override public HttpResponse auditAndHandle(HttpRequest request) { try { - switch (request.getMethod()) { - case GET: return get(request); - case POST: return post(request); - case DELETE: return delete(request); - case PATCH: return patch(request); - default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); - } + return switch (request.getMethod()) { + case GET -> get(request); + case POST -> post(request); + case DELETE -> delete(request); + case PATCH -> patch(request); + default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); + }; } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { @@ -159,8 +164,16 @@ public class OsApiHandler extends AuditLoggingRequestHandler { currentVersionObject.setString("version", osVersion.version().toFullString()); Optional<OsVersionTarget> target = targets.stream().filter(t -> t.osVersion().equals(osVersion)).findFirst(); currentVersionObject.setBool("targetVersion", target.isPresent()); - target.ifPresent(t -> currentVersionObject.setString("upgradeBudget", t.upgradeBudget().toString())); - target.ifPresent(t -> currentVersionObject.setLong("scheduledAt", t.scheduledAt().toEpochMilli())); + target.ifPresent(t -> { + currentVersionObject.setString("upgradeBudget", t.upgradeBudget().toString()); + currentVersionObject.setLong("scheduledAt", t.scheduledAt().toEpochMilli()); + Optional<Change> nextChange = osUpgradeScheduler.changeIn(t.osVersion().cloud()); + nextChange.ifPresent(c -> { + currentVersionObject.setString("nextVersion", c.version().toFullString()); + currentVersionObject.setLong("nextScheduledAt", c.scheduleAt().toEpochMilli()); + }); + }); + currentVersionObject.setString("cloud", osVersion.cloud().value()); Cursor nodesArray = currentVersionObject.setArray("nodes"); nodeVersions.forEach(nodeVersion -> { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java index 9268ea5ca1c..fac15cd23c4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -14,9 +14,12 @@ import org.junit.jupiter.api.Test; import java.time.Duration; import java.time.Instant; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Map; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -57,9 +60,9 @@ public class OsUpgradeSchedulerTest { tester.clock().advance(Duration.ofDays(30)); scheduler.maintain(); assertEquals(version0, - tester.controller().osVersionTarget(cloud).get().osVersion().version(), - "Target is unchanged because we're outside trigger period"); - tester.clock().advance(Duration.ofHours(7)); // Put us inside the trigger period + tester.controller().osVersionTarget(cloud).get().osVersion().version(), + "Target is unchanged because we're outside trigger period"); + tester.clock().advance(Duration.ofHours(7).plusMinutes(5)); // Put us inside the trigger period scheduler.maintain(); assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version(), @@ -69,11 +72,19 @@ public class OsUpgradeSchedulerTest { tester.clock().advance(Duration.ofDays(2)); scheduler.maintain(); assertEquals(version1, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + + // Estimate next change + Optional<OsUpgradeScheduler.Change> nextChange = scheduler.changeIn(cloud); + assertTrue(nextChange.isPresent()); + assertEquals("7.0.0.20220425", nextChange.get().version().toFullString()); + assertEquals("2022-05-02T07:00:00", LocalDateTime.ofInstant(nextChange.get().scheduleAt(), ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_DATE_TIME)); } @Test void schedule_stable_release() { ControllerTester tester = new ControllerTester(); + OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); Instant t0 = Instant.parse("2021-06-21T07:00:00.00Z"); // Inside trigger period tester.clock().setInstant(t0); @@ -86,19 +97,23 @@ public class OsUpgradeSchedulerTest { Version version1 = Version.fromString("8.1"); tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.stable, tester.clock().instant())); - scheduleUpgradeAfter(Duration.ZERO, version1, tester); + scheduleUpgradeAfter(Duration.ZERO, version1, scheduler, tester); // A newer version is triggered manually Version version3 = Version.fromString("8.3"); tester.controller().upgradeOsIn(cloud, version3, Duration.ZERO, false); // Nothing happens in next iteration as tagged release is older than manually triggered version - scheduleUpgradeAfter(Duration.ofDays(7), version3, tester); + scheduleUpgradeAfter(Duration.ofDays(7), version3, scheduler, tester); + + // Next change cannot be estimated for tagged releases + assertTrue(scheduler.changeIn(cloud).isEmpty(), "Next change is unknown"); } @Test void schedule_latest_release_in_cd() { ControllerTester tester = new ControllerTester(SystemName.cd); + OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); Instant t0 = Instant.parse("2021-06-21T07:00:00.00Z"); // Inside trigger period tester.clock().setInstant(t0); @@ -111,10 +126,10 @@ public class OsUpgradeSchedulerTest { Version version1 = Version.fromString("8.1"); tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.latest, tester.clock().instant())); - scheduleUpgradeAfter(Duration.ZERO, version0, tester); + scheduleUpgradeAfter(Duration.ZERO, version0, scheduler, tester); // Cooldown period passes and latest release is scheduled - scheduleUpgradeAfter(Duration.ofDays(1), version1, tester); + scheduleUpgradeAfter(Duration.ofDays(1), version1, scheduler, tester); } @Test @@ -135,9 +150,9 @@ public class OsUpgradeSchedulerTest { }); } - private void scheduleUpgradeAfter(Duration duration, Version version, ControllerTester tester) { + private void scheduleUpgradeAfter(Duration duration, Version version, OsUpgradeScheduler scheduler, ControllerTester tester) { tester.clock().advance(duration); - new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)).maintain(); + scheduler.maintain(); CloudName cloud = tester.controller().clouds().iterator().next(); OsVersionTarget target = tester.controller().osVersionTarget(cloud).get(); assertEquals(version, target.osVersion().version()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index 6ddc58feaea..15f0100ade8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.os; import com.yahoo.application.container.handler.Request; +import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.UpgradePolicy; @@ -11,10 +12,10 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.OsRelease; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintainer; import com.yahoo.vespa.hosted.controller.maintenance.OsUpgrader; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; @@ -57,10 +58,13 @@ public class OsApiTest extends ControllerContainerTest { tester = new ContainerTester(container, responses); tester.serviceRegistry().clock().setInstant(Instant.ofEpochMilli(1234)); addUserToHostedOperatorRole(operator); - zoneRegistryMock().setZones(zone1, zone2, zone3) - .reprovisionToUpgradeOsIn(zone3) - .setOsUpgradePolicy(cloud1, UpgradePolicy.builder().upgrade(zone1).upgrade(zone2).build()) - .setOsUpgradePolicy(cloud2, UpgradePolicy.builder().upgrade(zone3).build()); + tester.serviceRegistry().zoneRegistry().setZones(zone1, zone2, zone3) + .reprovisionToUpgradeOsIn(zone3) + .setOsUpgradePolicy(cloud1, UpgradePolicy.builder().upgrade(zone1).upgrade(zone2).build()) + .setOsUpgradePolicy(cloud2, UpgradePolicy.builder().upgrade(zone3).build()); + tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(Version.fromString("7.0"), + OsRelease.Tag.latest, + Instant.EPOCH)); osUpgraders = List.of( new OsUpgrader(tester.controller(), Duration.ofDays(1), cloud1), @@ -160,10 +164,6 @@ public class OsApiTest extends ControllerContainerTest { updateVersionStatus(); } - private ZoneRegistryMock zoneRegistryMock() { - return tester.serviceRegistry().zoneRegistry(); - } - private NodeRepositoryMock nodeRepository() { return tester.serviceRegistry().configServerMock().nodeRepository(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json index a5af4f45370..be94b85f113 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json @@ -104,6 +104,8 @@ "targetVersion": true, "upgradeBudget": "PT24H", "scheduledAt": 1234, + "nextVersion": "8.2.1.20211227", + "nextScheduledAt": 7200000, "cloud": "cloud2", "nodes": [ { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 51c4c893401..8e06cde420e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -472,6 +472,13 @@ public class Flags { APPLICATION_ID,HOSTNAME,NODE_TYPE,TENANT_ID,VESPA_VERSION ); + public static final UnboundBooleanFlag CLEANUP_TENANT_ROLES = defineFeatureFlag( + "cleanup-tenant-roles", false, + List.of("olaa"), "2022-08-10", "2022-10-01", + "Determines whether old tenant roles should be deleted", + "Takes effect next maintenance run" + ); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/tenant-cd-api/abi-spec.json b/tenant-cd-api/abi-spec.json index 11ec9f73be2..f072d9de169 100644 --- a/tenant-cd-api/abi-spec.json +++ b/tenant-cd-api/abi-spec.json @@ -15,6 +15,70 @@ ], "fields": [] }, + "ai.vespa.hosted.cd.DisabledInInstances": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [ + "public abstract java.lang.String[] value()" + ], + "fields": [] + }, + "ai.vespa.hosted.cd.DisabledInRegions": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [ + "public abstract java.lang.String[] value()" + ], + "fields": [] + }, + "ai.vespa.hosted.cd.EnabledInInstances": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [ + "public abstract java.lang.String[] value()" + ], + "fields": [] + }, + "ai.vespa.hosted.cd.EnabledInRegions": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [ + "public abstract java.lang.String[] value()" + ], + "fields": [] + }, "ai.vespa.hosted.cd.Endpoint": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/DisabledInInstances.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/DisabledInInstances.java new file mode 100644 index 00000000000..4a14509459a --- /dev/null +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/DisabledInInstances.java @@ -0,0 +1,43 @@ +package ai.vespa.hosted.cd; + +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.platform.commons.util.AnnotationUtils; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.List; +import java.util.Optional; + +/** + * @author jonmv + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE, ElementType.ANNOTATION_TYPE}) +@ExtendWith(DisabledInInstancesCondition.class) +public @interface DisabledInInstances { + + /** One or more instances that this should be disabled in. */ + String[] value(); + +} + +class DisabledInInstancesCondition implements ExecutionCondition { + + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { + Optional<DisabledInInstances> annotation = AnnotationUtils.findAnnotation(context.getElement(), DisabledInInstances.class); + if (annotation.isEmpty()) + return ConditionEvaluationResult.enabled(DisabledInInstances.class.getSimpleName() + " is not present"); + + List<String> disablingInstances = List.of(annotation.get().value()); + String thisInstance = TestRuntime.get().application().instance(); + String reason = "Disabled in: %s. Current instance: %s.".formatted(disablingInstances.isEmpty() ? "no instances" : "instances " + String.join(", ", disablingInstances), thisInstance); + return disablingInstances.contains(thisInstance) ? ConditionEvaluationResult.disabled(reason) : ConditionEvaluationResult.enabled(reason); + } + +}
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/DisabledInRegions.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/DisabledInRegions.java new file mode 100644 index 00000000000..aeb6a001726 --- /dev/null +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/DisabledInRegions.java @@ -0,0 +1,44 @@ +package ai.vespa.hosted.cd; + +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.platform.commons.util.AnnotationUtils; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; + +/** + * @author jonmv + */ +@Target({ElementType.METHOD, ElementType.TYPE, ElementType.ANNOTATION_TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(DisabledInRegionsCondition.class) +public @interface DisabledInRegions { + + /** One or more regions that this should be disabled in. */ + String[] value(); + +} + +class DisabledInRegionsCondition implements ExecutionCondition { + + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { + Optional<DisabledInRegions> annotation = AnnotationUtils.findAnnotation(context.getElement(), DisabledInRegions.class); + if (annotation.isEmpty()) + return ConditionEvaluationResult.enabled(DisabledInRegions.class.getSimpleName() + " is not present"); + + List<String> disablingRegions = List.of(annotation.get().value()); + String thisRegion = TestRuntime.get().application().instance(); + String reason = "Disabled in: %s. Current region: %s.".formatted(disablingRegions.isEmpty() ? "no regions" : "regions " + String.join(", ", disablingRegions), thisRegion); + return disablingRegions.contains(thisRegion) ? ConditionEvaluationResult.disabled(reason) : ConditionEvaluationResult.enabled(reason); + } + +}
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/EnabledInInstances.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/EnabledInInstances.java new file mode 100644 index 00000000000..dfe22dacb11 --- /dev/null +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/EnabledInInstances.java @@ -0,0 +1,43 @@ +package ai.vespa.hosted.cd; + +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.platform.commons.util.AnnotationUtils; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.List; +import java.util.Optional; + +/** + * @author jonmv + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE, ElementType.ANNOTATION_TYPE}) +@ExtendWith(EnabledInInstancesCondition.class) +public @interface EnabledInInstances { + + /** One or more instances that this should be enabled in. */ + String[] value(); + +} + +class EnabledInInstancesCondition implements ExecutionCondition { + + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { + Optional<EnabledInInstances> annotation = AnnotationUtils.findAnnotation(context.getElement(), EnabledInInstances.class); + if (annotation.isEmpty()) + return ConditionEvaluationResult.enabled(EnabledInInstances.class.getSimpleName() + " is not present"); + + List<String> enablingInstances = List.of(annotation.get().value()); + String thisInstance = TestRuntime.get().application().instance(); + String reason = "Enabled in: %s. Current instance: %s.".formatted(enablingInstances.isEmpty() ? "no instances" : "instances " + String.join(", ", enablingInstances), thisInstance); + return enablingInstances.contains(thisInstance) ? ConditionEvaluationResult.enabled(reason) : ConditionEvaluationResult.disabled(reason); + } + +}
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/EnabledInRegions.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/EnabledInRegions.java new file mode 100644 index 00000000000..db2e5ac5f95 --- /dev/null +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/EnabledInRegions.java @@ -0,0 +1,43 @@ +package ai.vespa.hosted.cd; + +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.platform.commons.util.AnnotationUtils; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.List; +import java.util.Optional; + +/** + * @author jonmv + */ +@Target({ElementType.METHOD, ElementType.TYPE, ElementType.ANNOTATION_TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(EnabledInRegionsCondition.class) +public @interface EnabledInRegions { + + /** One or more regions that this should be enabled in. */ + String[] value(); + +} + +class EnabledInRegionsCondition implements ExecutionCondition { + + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { + Optional<EnabledInRegions> annotation = AnnotationUtils.findAnnotation(context.getElement(), EnabledInRegions.class); + if (annotation.isEmpty()) + return ConditionEvaluationResult.enabled(EnabledInRegions.class.getSimpleName() + " is not present"); + + List<String> enablingRegions = List.of(annotation.get().value()); + String thisRegion = TestRuntime.get().application().instance(); + String reason = "Enabled in: %s. Current region: %s.".formatted(enablingRegions.isEmpty() ? "no regions" : "regions " + String.join(", ", enablingRegions), thisRegion); + return enablingRegions.contains(thisRegion) ? ConditionEvaluationResult.enabled(reason) : ConditionEvaluationResult.disabled(reason); + } + +}
\ No newline at end of file diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index d7ef20c31c8..fb0e79b6695 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -436,6 +436,13 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { return QuotaUsage.calculateUsage(usageEntity, quotaEntity); } + @Override + public void deleteSubdomain(AthenzDomain parent, String name) { + URI uri = zmsUrl.resolve(String.format("subdomain/%s/%s", parent.getName(), name)); + HttpUriRequest request = RequestBuilder.delete(uri).build(); + execute(request, response -> readEntity(response, Void.class)); + } + public AthenzRoleInformation getFullRoleInformation(AthenzRole role) { var uri = zmsUrl.resolve(String.format("domain/%s/role/%s?pending=true&auditLog=true", role.domain().getName(), role.roleName())); var request = RequestBuilder.get(uri).build(); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index e15af58cb76..983924eca6b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -89,5 +89,7 @@ public interface ZmsClient extends Closeable { QuotaUsage getQuotaUsage(); + void deleteSubdomain(AthenzDomain parent, String name); + void close(); } |