diff options
| author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-09-30 17:09:59 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-09-30 17:09:59 +0200 |
| commit | c75ff9433d7d8e7d1a0186592b57597ae632581c (patch) | |
| tree | 9bed7a0bcca7752796a6c0bf2ead31694fcb4037 /container-search/src/main | |
| parent | 041de033862d7e9ee8ff16770cd5cc63a11b5d16 (diff) | |
| parent | 253fe67dfae65ab11ed008590403ffbe10fab02e (diff) | |
Merge branch 'master' into balder/do-not-depend-on-clusterinfo
Diffstat (limited to 'container-search/src/main')
270 files changed, 4627 insertions, 4704 deletions
diff --git a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 index 639ff8255d2..c0cf293f2ea 100644 --- a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 +++ b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 @@ -18,22 +18,9 @@ options { protected Stack<expression_scope> expression_stack = new Stack(); } -// tokens for command syntax - CREATE : 'create'; +// tokens + SELECT : 'select'; - INSERT : 'insert'; - UPDATE : 'update'; - SET : 'set'; - VIEW : 'view'; - TABLE : 'table'; - DELETE : 'delete'; - INTO : 'into'; - VALUES : 'values'; - IMPORT : 'import'; - NEXT : 'next'; - PAGED : 'paged'; - FALLBACK : 'fallback'; - IMPORT_FROM :; LIMIT : 'limit'; OFFSET : 'offset'; @@ -44,36 +31,11 @@ options { FROM : 'from'; SOURCES : 'sources'; AS : 'as'; - MERGE : 'merge'; - LEFT : 'left'; - JOIN : 'join'; - - ON : 'on'; + COMMA : ','; OUTPUT : 'output'; COUNT : 'count'; - RETURNING : 'returning'; - APPLY : 'apply'; - CAST : 'cast'; - - BEGIN : 'begin'; - END : 'end'; - - // type-related - TYPE_BYTE : 'byte'; - TYPE_INT16 : 'int16'; - TYPE_INT32 : 'int32'; - TYPE_INT64 : 'int64'; - TYPE_STRING : 'string'; - TYPE_DOUBLE : 'double'; - TYPE_TIMESTAMP : 'timestamp'; - TYPE_BOOLEAN : 'boolean'; - TYPE_ARRAY : 'array'; - TYPE_MAP : 'map'; - - // READ_FIELD; - - // token literals + TRUE : 'true'; FALSE : 'false'; @@ -123,76 +85,14 @@ options { // statement delimiter SEMI : ';'; - PROGRAM : 'program'; TIMEOUT : 'timeout'; -//following node names seems only used for root node name -//not exactly matching anything, should we still keep it? -// Follow tree name are not no longer used -//comment out maybe remove -// ARGUMENT : ; -// TYPE : ; -// NAME : ; -// DEFAULT :; -// -// ANNOTATE : ; -// -// PROJECT :; -// FILTER :; -// -// PROPERTY :; -// LITERAL :; -// PARAMETER :; -// SCALAR_LITERAL :; -// MAP_LITERAL :; -// // ARRAY_LITERAL :; -// FIELD :; -// //* FIELD_ASSIGNMENT; -// ALL_SOURCE :; -// MULTI_SOURCE :; -// EXPRESSION_SOURCE :; -// SEQUENCE_SOURCE :; -// CALL_SOURCE :; -// PIPELINE_STEP :; -// AUTO_ALIAS :; -// // BINOP_ADD :; -// // BINOP_SUB :; -// BINOP_MULT :; -// BINOP_DIV :; -// BINOP_MOD :; -// -// UNOP_MINUS :; -// UNOP_NOT :; -// -// STATEMENT_SELECTVAR :; -// STATEMENT_QUERY :; -// -// FIELDREF :; -// PATH :; -// -// CALL :; -// -// // INDEXREF :; -// PROPERTYREF :; -// -// LEFT_JOIN :; -// RIGHT_JOIN :; -//// FULL_JOIN; -// -// ALIAS :; -// -// INSERT_VALUES :; -// UPDATE_VALUES :; -// -// NO_ARGUMENTS :; - - /*------------------------------------------------------------------ * LEXER RULES *------------------------------------------------------------------*/ -ID : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_'|':')* +ID : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_'|':'|'-')* ; LONG_INT : '-'?'0'..'9'+ ('L'|'l') @@ -210,7 +110,6 @@ FLOAT fragment EXPONENT : ('e'|'E') ('+'|'-')? ('0'..'9')+ ; - fragment DIGIT : '0'..'9' ; @@ -220,13 +119,10 @@ LETTER : 'a'..'z' | 'A'..'Z' ; -//STRING : DQ ( ESC_SEQ | ~('\\'| DQ) )* DQ -// | SQ ( ESC_SEQ | ~('\\' | SQ) )* SQ STRING : '"' ( ESC_SEQ | ~('\\'| '"') )* '"' | '\'' ( ESC_SEQ | ~('\\' | '\'') )* '\'' ; -///////////////////////////// fragment HEX_DIGIT : ('0'..'9'|'a'..'f'|'A'..'F') ; @@ -235,18 +131,8 @@ fragment ESC_SEQ : '\\' ('b'|'t'|'n'|'f'|'r'|'\"'|'\''|'\\'|'/') | UNICODE_ESC - // | OCTAL_ESC ; -/* -fragment -OCTAL_ESC - : '\\' ('0'..'3') ('0'..'7') ('0'..'7') - | '\\' ('0'..'7') ('0'..'7') - | '\\' ('0'..'7') - ; -*/ - fragment UNICODE_ESC : '\\' 'u' HEX_DIGIT HEX_DIGIT HEX_DIGIT HEX_DIGIT @@ -256,17 +142,11 @@ WS : ( ' ' | '\t' | '\r' | '\n' - // ) {$channel=HIDDEN;} ) -> channel(HIDDEN) ; -//COMMENT -// : ('//') ~('\n'|'\r')* '\r'? '\n'? {$channel=HIDDEN;} -// | '/*' ( options {greedy=false;} : . )* '*/' {$channel=HIDDEN;} -// ; COMMENT : ( ('//') ~('\n'|'\r')* '\r'? '\n'? - // | '/*' ( options {greedy=false;} : . )* '*/' | '/*' .*? '*/' ) -> channel(HIDDEN) @@ -285,56 +165,20 @@ VESPA_GROUPING_ARG (')' | ']' | '>') ; -/*------------------------------------LPAREN! select_statement RPAREN!------------------------------ - * PARSER RULES - *------------------------------------------------------------------*/ +// --------- parser rules ------------ ident : keyword_as_ident //{addChild(new TerminalNodeImpl(keyword_as_ident.getText()));} - //{return ID<IDNode>[$keyword_as_ident.text];} | ID ; keyword_as_ident - : SELECT | TABLE | DELETE | INTO | VALUES | LIMIT | OFFSET | WHERE | 'order' | 'by' | DESC | MERGE | LEFT | JOIN - | ON | OUTPUT | COUNT | BEGIN | END | APPLY | TYPE_BYTE | TYPE_INT16 | TYPE_INT32 | TYPE_INT64 | TYPE_BOOLEAN | TYPE_TIMESTAMP | TYPE_DOUBLE | TYPE_STRING | TYPE_ARRAY | TYPE_MAP - | VIEW | CREATE | IMPORT | PROGRAM | NEXT | PAGED | SOURCES | SET | MATCHES | LIKE + : SELECT | LIMIT | OFFSET | WHERE | 'order' | 'by' | DESC | OUTPUT | COUNT | SOURCES | MATCHES | LIKE ; -//program : params? (import_statement SEMI)* (ddl SEMI)* (statement SEMI)* EOF -// -> ^(PROGRAM params? import_statement* ddl* statement*) -// ; -program : params? (import_statement SEMI)* (ddl SEMI)* (statement SEMI)* EOF +program : (statement SEMI)* EOF ; -//params -// : PROGRAM! LPAREN! program_arglist? RPAREN! SEMI! -// ; -params - : PROGRAM LPAREN program_arglist? RPAREN SEMI - ; - -//import_statement -// : IMPORT moduleName AS moduleId -> ^(IMPORT moduleName moduleId) -// | IMPORT moduleId -> ^(IMPORT moduleId) -// | FROM moduleName IMPORT import_list -> ^(IMPORT_FROM moduleName import_list+) -// ; -import_statement - : IMPORT moduleName AS moduleId - | IMPORT moduleId - | FROM moduleName IMPORT import_list - ; - -//import_list -// : moduleId (',' moduleId)* -> moduleId+ -// ; -import_list - : moduleId (',' moduleId)* - ; - -//moduleId -// : ID -> STRING<LiteralNode>[$ID, $ID.text] -// ; moduleId : ID ; @@ -344,72 +188,18 @@ moduleName | namespaced_name ; -ddl - : view - ; - -//view : CREATE VIEW ID AS source_statement -> ^(VIEW ID source_statement) -// ; -view : CREATE VIEW ID AS source_statement - ; - - -//program_arglist -// : procedure_argument (',' procedure_argument)* -> procedure_argument+ -// ; -program_arglist - : procedure_argument (',' procedure_argument)* - ; - -//procedure_argument -// : AT (ident TYPE_ARRAY LT typename GTEQ (expression[false])? ) {registerParameter($ident.tree, $typename.tree);} -> ^(ARGUMENT ident typename expression?) -// | AT (ident typename ('=' expression[false])? ) {registerParameter($ident.tree, $typename.tree);} -> ^(ARGUMENT ident typename expression?) -// ; -procedure_argument - : - AT (ident TYPE_ARRAY LT typename GTEQ (expression[false])? ) {registerParameter($ident.start.getText(), $typename.start.getText());} - | AT (ident typename ('=' expression[false])? ) {registerParameter($ident.start.getText(), $typename.start.getText());} - ; - statement : output_statement - | selectvar_statement - | next_statement ; -//output_statement -// : source_statement paged_clause? output_spec? -> ^(STATEMENT_QUERY source_statement paged_clause? output_spec?) -// ; output_statement - : source_statement paged_clause? output_spec? - ; - -//paged_clause -// : PAGED^ fixed_or_parameter -// ; -paged_clause - : PAGED fixed_or_parameter + : source_statement output_spec? ; -//next_statement -// : NEXT^ literalString OUTPUT! AS! ident -// ; -next_statement - : NEXT literalString OUTPUT AS ident - ; - -//source_statement -// : query_statement (PIPE pipeline_step)+ -> ^(PIPE query_statement pipeline_step+) -// | query_statement -// ; source_statement : query_statement (PIPE pipeline_step)* ; - -//pipeline_step -// : namespaced_name arguments[false]? -> ^(PIPELINE_STEP namespaced_name arguments?) -// ; pipeline_step : namespaced_name arguments[false]? | vespa_grouping @@ -420,78 +210,19 @@ vespa_grouping | annotation VESPA_GROUPING ; -//selectvar_statement -// : CREATE ('temp' | 'temporary') TABLE ident AS LPAREN source_statement RPAREN -> ^(STATEMENT_SELECTVAR ident source_statement) -// ; -selectvar_statement - : CREATE ('temp' | 'temporary') TABLE ident AS LPAREN source_statement RPAREN - ; - - -typename - : TYPE_BYTE | TYPE_INT16 | TYPE_INT32 | TYPE_INT64 | TYPE_STRING | TYPE_BOOLEAN | TYPE_TIMESTAMP - | arrayType | mapType | TYPE_DOUBLE - ; - -//arrayType -// : TYPE_ARRAY^ LT! typename GT! -// ; -arrayType - : TYPE_ARRAY LT typename GT - ; - -//mapType -// : TYPE_MAP^ LT! typename GT! -// ; -mapType - : TYPE_MAP LT typename GT - ; - -//output_spec -// : (OUTPUT^ AS! ident) -// | (OUTPUT! COUNT^ AS! ident) -// ; output_spec : (OUTPUT AS ident) | (OUTPUT COUNT AS ident) ; - query_statement - : merge_statement - | select_statement - | insert_statement - | delete_statement - | update_statement + : select_statement ; -// This does not use the UNION / UNION ALL from SQL because the semantics are different than SQL UNION -// - no set operation is implied (no DISTINCT) -// - CQL resultsets may be heterogeneous (rows may have heterogenous types) -//merge_statement -// : merge_component (MERGE merge_component)+ -> ^(MERGE merge_component+) -// ; -merge_statement - : merge_component (MERGE merge_component)+ - ; - -merge_component - : select_statement - | LPAREN source_statement RPAREN - ; - -//select_statement -// : SELECT^ select_field_spec select_source where? orderby? limit? offset? timeout? fallback? -// ; select_statement - : SELECT select_field_spec select_source? where? orderby? limit? offset? timeout? fallback? -// | SELECT select_field_spec where? orderby? limit? offset? timeout? fallback? + : SELECT select_field_spec select_source? where? orderby? limit? offset? timeout? ; -//select_field_spec -// : field_def (COMMA field_def)* -> ^(PROJECT field_def+) -// | STAR! -// ; select_field_spec : project_spec | STAR @@ -501,31 +232,14 @@ project_spec : field_def (COMMA field_def)* ; -//fallback -// : FALLBACK^ select_statement -// ; -fallback - : FALLBACK select_statement - ; - -//timeout -// : TIMEOUT^ fixed_or_parameter -// ; timeout : TIMEOUT fixed_or_parameter ; -//select_source -// : FROM SOURCES STAR -> ALL_SOURCE -// | FROM SOURCES source_list -> ^(MULTI_SOURCE source_list) -// | FROM^ source_spec join_expr* -// | -> EXPRESSION_SOURCE -// ; - select_source : select_source_all | select_source_multi - | select_source_join + | select_source_from ; select_source_all @@ -536,53 +250,22 @@ select_source_multi : FROM SOURCES source_list ; -select_source_join - : FROM source_spec join_expr* +select_source_from + : FROM source_spec ; -//source_list -// : namespaced_name (COMMA namespaced_name)* -> namespaced_name+ -// ; source_list : namespaced_name (COMMA namespaced_name )* ; -//join_expr -// : (join_spec^ source_spec ON! joinExpression) -// ; -join_expr - : (join_spec source_spec ON joinExpression) - ; - -//join_spec -// : LEFT JOIN -> LEFT_JOIN -// | 'inner'? JOIN -> JOIN -// ; -join_spec - : LEFT JOIN - | 'inner'? JOIN - ; - -//source_spec -// : ( data_source (alias_def { ($data_source.tree).addChild($alias_def.tree); } )? ) -> data_source -// ; source_spec : ( data_source (alias_def { ($data_source.ctx).addChild($alias_def.ctx); })? ) ; -//alias_def -// : (AS? ID) -> ^(ALIAS ID) -// ; alias_def : (AS? ID) ; -//data_source -// : namespaced_name arguments[true]? -> ^(CALL_SOURCE namespaced_name arguments?) -// | LPAREN! source_statement RPAREN! -// | AT ident -> ^(SEQUENCE_SOURCE ident) -// ; - data_source : call_source | LPAREN source_statement RPAREN @@ -597,84 +280,47 @@ sequence_source : AT ident ; -//namespaced_name -// : (ident (DOT ident)* (DOT STAR)?) -> ^(PATH ident+ STAR?) -// ; namespaced_name : (ident (DOT ident)* (DOT STAR)?) ; -//orderby -// : ORDERBY orderby_fields -> ^(ORDERBY orderby_fields) -// ; orderby : ORDERBY orderby_fields ; -//orderby_fields -// : orderby_field (COMMA orderby_field)* -> orderby_field+ -// ; orderby_fields : orderby_field (COMMA orderby_field)* ; -//orderby_field -// : expression[true] DESC -> ^(DESC expression) -// | expression[true] ('asc')? -> ^(ASC expression) -// ; orderby_field : expression[true] DESC | expression[true] ('asc')? ; -//limit -// : LIMIT fixed_or_parameter -> ^(LIMIT fixed_or_parameter) -// ; limit : LIMIT fixed_or_parameter ; -//offset -// : OFFSET fixed_or_parameter -> ^(OFFSET fixed_or_parameter) -// ; offset : OFFSET fixed_or_parameter ; - -//where -// : WHERE expression[true] -> ^(FILTER expression) -// ; where : WHERE expression[true] ; -//field_def -// : expression[true] alias_def? -> ^(FIELD expression alias_def?) -// ; field_def : expression[true] alias_def? ; -// Hive doesn't have syntax for creating maps - this is an extension -//mapExpression -// : LBRACE i+=propertyNameAndValue? (COMMA i+=propertyNameAndValue)* RBRACE -> ^(MAP_LITERAL $i*) -// ; mapExpression : LBRACE propertyNameAndValue? (COMMA propertyNameAndValue)* RBRACE ; -//constantMapExpression -// : LBRACE i+=constantPropertyNameAndValue? (COMMA i+=constantPropertyNameAndValue)* RBRACE -> ^(MAP_LITERAL $i+) -// ; constantMapExpression : LBRACE constantPropertyNameAndValue? (COMMA constantPropertyNameAndValue)* RBRACE ; -//arguments[boolean in_select] -// : LPAREN RPAREN -> NO_ARGUMENTS -// | LPAREN (argument[$in_select] (COMMA argument[$in_select])*) RPAREN -> argument+ -// ; arguments[boolean in_select] : LPAREN RPAREN | LPAREN (argument[$in_select] (COMMA argument[$in_select])*) RPAREN @@ -684,45 +330,9 @@ argument[boolean in_select] : expression[$in_select] ; -// -------- join expressions ------------ - -// Limit expression syntax for joins -// for now, a single equality test and one field from each source -// this makes it easier for the prototype to be sure it can generate code -// effectively this means it can always turn the join into a query to one source, collecting all of the -// keys from the results, and then a query to the other source -// (or querying the other source inline) -// do not support map or index references -// this can become more general as the engine gets more capable - -//joinExpression -// : joinDereferencedExpression EQ^ joinDereferencedExpression -// ; -joinExpression - : joinDereferencedExpression EQ joinDereferencedExpression - ; - -//joinDereferencedExpression -// : (namespaced_name -> ^(FIELDREF namespaced_name)) -// ; -joinDereferencedExpression - : namespaced_name - ; - // --------- expressions ------------ -//expression[boolean select] -//scope { -// boolean in_select; -//} -//@init { -// $expression::in_select = $select; -//} -// : ( annotation logicalORExpression) -> ^(ANNOTATE annotation logicalORExpression) -// | logicalORExpression -// ; - -expression [boolean select] +expression [boolean select] @init { expression_stack.push(new expression_scope()); expression_stack.peek().in_select = select; @@ -739,50 +349,30 @@ nullOperator : 'null' ; -//annotation -// : LBRACKET! constantMapExpression RBRACKET! -// ; annotateExpression : annotation logicalORExpression ; + annotation : LBRACKET constantMapExpression RBRACKET ; -//logicalORExpression -// : logicalANDExpression (OR logicalANDExpression)+ -> ^(OR logicalANDExpression+) -// | logicalANDExpression -// ; logicalORExpression : logicalANDExpression (OR logicalANDExpression)+ | logicalANDExpression ; -//logicalANDExpression -// : equalityExpression (AND equalityExpression)+ -> ^(AND equalityExpression+) -// | equalityExpression -// ; logicalANDExpression : equalityExpression (AND equalityExpression)* ; -//equalityExpression -// : relationalExpression ( ((IN^ | NOT_IN^) inNotInTarget) -// | (IS_NULL^ | IS_NOT_NULL^) -// | (equalityOp^ relationalExpression) )? -// ; - -equalityExpression //changed for parsing literal tests +equalityExpression : relationalExpression ( ((IN | NOT_IN) inNotInTarget) | (IS_NULL | IS_NOT_NULL) | (equalityOp relationalExpression) ) | relationalExpression ; -//inNotInTarget -// : {$expression::in_select}? LPAREN select_statement RPAREN -> ^(QUERY_ARRAY select_statement) -// | literal_list -// ; inNotInTarget : {expression_stack.peek().in_select}? LPAREN select_statement RPAREN | literal_list @@ -792,9 +382,6 @@ equalityOp : (EQ | NEQ | LIKE | NOTLIKE | MATCHES | NOTMATCHES | CONTAINS) ; -//relationalExpression -// : additiveExpression (relationalOp^ additiveExpression)* -// ; relationalExpression : additiveExpression (relationalOp additiveExpression)? ; @@ -803,64 +390,35 @@ relationalOp : (LT | GT | LTEQ | GTEQ) ; -//additiveExpression -// : multiplicativeExpression (additiveOp^ additiveExpression)? -// ; additiveExpression : multiplicativeExpression (additiveOp additiveExpression)? ; -//additiveOp -// : '+' -> BINOP_ADD -// | '-' -> BINOP_SUB -// ; additiveOp : '+' | '-' ; -//multiplicativeExpression -// : unaryExpression (multOp^ multiplicativeExpression)? -// ; multiplicativeExpression : unaryExpression (multOp multiplicativeExpression)? ; -//multOp : '*' -> BINOP_MULT -// | '/' -> BINOP_DIV -// | '%' -> BINOP_MOD -// ; -multOp +multOp : '*' | '/' | '%' ; -//unaryOp -// : '-' -> UNOP_MINUS -// | '!' -> UNOP_NOT -// ; unaryOp : '-' | '!' ; -//unaryExpression -// : dereferencedExpression -// | unaryOp^ dereferencedExpression -// ; unaryExpression : dereferencedExpression | unaryOp dereferencedExpression ; -//dereferencedExpression -// : (primaryExpression -> primaryExpression) -// ( -// (LBRACKET idx=expression[$expression::in_select] RBRACKET) -> ^(INDEXREF $dereferencedExpression $idx) -// | (DOT nm=ID) -> ^(PROPERTYREF $dereferencedExpression $nm) -// )* -// ; dereferencedExpression @init{ boolean in_select = expression_stack.peek().in_select; @@ -878,43 +436,12 @@ indexref[boolean in_select] propertyref : DOT nm=ID ; -//operatorCall -// : multOp arguments[$expression::in_select] -> ^(multOp arguments) -// | additiveOp arguments[$expression::in_select] -> ^(additiveOp arguments) -// | AND arguments[$expression::in_select] -> ^(AND arguments) -// | OR arguments[$expression::in_select] -> ^(OR arguments) -// ; -operatorCall -@init{ - boolean in_select = expression_stack.peek().in_select; -} - : multOp arguments[in_select] - | additiveOp arguments[in_select] - | AND arguments[in_select] - | OR arguments[in_select] - ; - -// TODO: temporarily disable CAST, need to think through how types are named - -//primaryExpression -// : namespaced_name arguments[$expression::in_select] -> ^(CALL namespaced_name arguments) -//// | CAST LPAREN expression[$expression::in_select] AS typename RPAREN -> ^(CAST expression typename) -// | APPLY! operatorCall -// | parameter -// | namespaced_name -> ^(FIELDREF namespaced_name) -// | scalar_literal -// | LBRACKET i+=expression[$expression::in_select]? (COMMA (i+=expression[$expression::in_select]))* RBRACKET -> ^(ARRAY_LITERAL $i*) -// | mapExpression -// | LPAREN! expression[$expression::in_select] RPAREN! -// ; primaryExpression @init { boolean in_select = expression_stack.peek().in_select; } : callExpresion[in_select] -// | CAST LPAREN expression[$expression::in_select] AS typename RPAREN -> ^(CAST expression typename) -// | APPLY operatorCall | parameter | fieldref | scalar_literal @@ -937,32 +464,19 @@ arrayLiteral : LBRACKET expression[in_select]? (COMMA expression[in_select])* RBRACKET ; -// a parameter is an argument from outside the script/program -//parameter -// : AT ident -> ^(PARAMETER ident) -// ; -parameter +// a parameter is an argument from outside the YQL statement +parameter : AT ident ; -//propertyNameAndValue -// : propertyName ':' expression[$expression::in_select] -> ^(PROPERTY propertyName expression) -// ; propertyNameAndValue : propertyName ':' expression[{expression_stack.peek().in_select}] //{return (PROPERTY propertyName expression);} ; -//constantPropertyNameAndValue -// : propertyName ':' constantExpression -> ^(PROPERTY propertyName constantExpression) -// ; constantPropertyNameAndValue : propertyName ':' constantExpression ; -//propertyName -// : ID -> STRING<LiteralNode>[$ID, $ID.text] -// | literalString -// ; propertyName : ID | literalString @@ -972,56 +486,32 @@ constantExpression : scalar_literal | constantMapExpression | constantArray + | parameter ; -//constantArray -// : LBRACKET i+=constantExpression? (COMMA i+=constantExpression)* RBRACKET -> ^(ARRAY_LITERAL $i*) -// ; constantArray : LBRACKET i+=constantExpression? (COMMA i+=constantExpression)* RBRACKET ; -// TODO: fix INT L and INT b parsing -- should not permit whitespace between them -//scalar_literal -// : TRUE -> TRUE<LiteralNode>[$TRUE, true] -// | FALSE -> FALSE<LiteralNode>[$FALSE, false] -// | literalString -// | LONG_INT -> INT<LiteralNode>[$LONG_INT, Long.parseLong($LONG_INT.text.substring(0, $LONG_INT.text.length()-1))] -//// | INT 'b' -> INT<LiteralNode>[$INT, Byte.parseByte($INT.text)] -// | INT -> INT<LiteralNode>[$INT, Integer.parseInt($INT.text)] -// | FLOAT -> FLOAT<LiteralNode>[$FLOAT, Double.parseDouble($FLOAT.text)] -// ; scalar_literal : TRUE | FALSE | STRING | LONG_INT -// | INT 'b' -> INT<LiteralNode>[$INT, Byte.parseByte($INT.text)] - | INT + | INT | FLOAT ; -//literalString -// : STRING -> STRING<LiteralNode>[$STRING, StringUnescaper.unquote($STRING.text)] -// ; literalString : STRING ; -//array_parameter -// : AT i=ident {isArrayParameter($i.text)}? -> ^(PARAMETER $i) -// ; array_parameter : AT i=ident {isArrayParameter($i.ctx)}? ; -// array literal for IN/NOT_IN using SQL syntax -//literal_list -// : LPAREN array_parameter RPAREN -> array_parameter -// | LPAREN literal_element (COMMA literal_element)* RPAREN -> ^(ARRAY_LITERAL literal_element+) -// ; literal_list - : LPAREN literal_element (COMMA literal_element)* RPAREN //{return ^(ARRAY_LITERAL literal_element+);} + : LPAREN literal_element (COMMA literal_element)* RPAREN ; literal_element @@ -1029,115 +519,7 @@ literal_element | parameter ; -//fixed_or_parameter -// : INT -> INT<LiteralNode>[$INT, Integer.parseInt($INT.text)] -// | parameter -// ; fixed_or_parameter : INT | parameter ; - - -// INSERT - -//insert_statement -// : INSERT^ insert_source insert_values returning_spec? -// ; -insert_statement - : INSERT insert_source insert_values returning_spec? - ; - -//insert_source -// : INTO^ write_data_source -// ; -insert_source - : INTO write_data_source - ; - -//write_data_source -// : namespaced_name -> ^(CALL_SOURCE namespaced_name) -// ; -write_data_source - : namespaced_name - ; - -//insert_values -// : field_names_spec VALUES field_values_group_spec (COMMA field_values_group_spec)* -> ^(INSERT_VALUES field_names_spec field_values_group_spec+) -// | query_statement -> ^(INSERT_QUERY query_statement) -// ; - -insert_values - : field_names_spec VALUES field_values_group_spec (COMMA field_values_group_spec)* - | query_statement - ; - -//field_names_spec -// : LPAREN field_def (COMMA field_def)* RPAREN -> field_def+ -// ; -field_names_spec - : LPAREN field_def (COMMA field_def)* RPAREN - ; - -//field_values_spec -// : LPAREN expression[true] (COMMA expression[true])* RPAREN -> expression+ -// ; -field_values_spec - : LPAREN expression[true] (COMMA expression[true])* RPAREN - ; - -//field_values_group_spec -// : LPAREN expression[true] (COMMA expression[true])* RPAREN -> ^(FIELD_VALUES_GROUP expression+) -// ; -field_values_group_spec - : LPAREN expression[true] (COMMA expression[true])* RPAREN - ; - -returning_spec - : RETURNING select_field_spec - ; - -// DELETE - -//delete_statement -// : DELETE^ delete_source where? returning_spec? -// ; -delete_statement - : DELETE delete_source where? returning_spec? - ; - -//delete_source -// : FROM^ write_data_source -// ; -delete_source - : FROM write_data_source - ; - -// UPDATE - -//update_statement -// : UPDATE^ update_source SET update_values where? returning_spec? -// ; -update_statement - : UPDATE update_source SET update_values where? returning_spec? - ; - -//update_source -// : namespaced_name -// ; -update_source - : write_data_source - ; - -//update_values -// : field_names_spec EQ field_values_spec -> ^(UPDATE_VALUES field_names_spec field_values_spec) -// | field_def (COMMA field_def)* -> ^(UPDATE_VALUES field_def+) -// ; -update_values - : field_names_spec EQ field_values_spec - | field_def (COMMA field_def)* - ; - - - - diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index 365ee299ca4..306c7c80577 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude; - import com.yahoo.language.process.StemMode; import java.util.ArrayList; @@ -10,7 +9,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; - /** * Information about configured settings of a field or field collection (an actual index or not) in a search definition. * There are two types of settings: @@ -25,32 +23,15 @@ import java.util.Set; */ public class Index { - public static class Attribute { - - private boolean tokenizedContent = false; - public final String name; - - public Attribute(String name) { - this.name = name; - } - - public boolean isTokenizedContent() { - return tokenizedContent; - } - - public void setTokenizedContent(boolean tokenizedContent) { - this.tokenizedContent = tokenizedContent; - } - } - /** The null index - don't use this for name lookups */ public static final Index nullIndex = new Index("(null)"); - private String name; + private final String name; - private List<String> aliases = new ArrayList<>(); + private final List<String> aliases = new ArrayList<>(); // The state resulting from adding commands to this (using addCommand) + private boolean tensor = false; private boolean uriIndex = false; private boolean hostIndex = false; private StemMode stemMode = StemMode.NONE; @@ -65,6 +46,7 @@ public class Index { private boolean normalize = false; private boolean literalBoost = false; private boolean numerical = false; + private boolean predicate = false; private long predicateUpperBound = Long.MAX_VALUE; private long predicateLowerBound = Long.MIN_VALUE; @@ -74,17 +56,17 @@ public class Index { private boolean isNGram = false; private int gramSize = 2; - /** Whether implicit phrases should lead to a phrase item or an and item */ - private boolean phraseSegmenting = true; + /** Whether implicit phrases should lead to a phrase item or an and item. */ + private Boolean phraseSegmenting = false; /** The string terminating an exact token in this index, or null to use the default (space) */ private String exactTerminator = null; - /** Commands which are not coverted into a field */ - private Set<String> commands = new java.util.HashSet<>(); + /** Commands which are not converted into a field */ + private final Set<String> commands = new java.util.HashSet<>(); /** All the commands added to this, including those converted to fields above */ - private List<String> allCommands = new java.util.ArrayList<>(); + private final List<String> allCommands = new java.util.ArrayList<>(); public Index(String name) { this.name = name; @@ -140,61 +122,71 @@ public class Index { } /** Adds a type or untyped command string to this */ - public Index addCommand(String commandString) { - allCommands.add(commandString); + public Index addCommand(String command) { + allCommands.add(command); - if ("fullurl".equals(commandString)) { + if (command.startsWith("type tensor(") || command.startsWith("type tensor<")) { // TODO: Type info can replace numerical, predicate, multivalue + setTensor(true); + } else if ("fullurl".equals(command)) { setUriIndex(true); - } else if ("urlhost".equals(commandString)) { + } else if ("urlhost".equals(command)) { setHostIndex(true); - } else if (commandString.startsWith("stem ")) { - setStemMode(commandString.substring(5)); - } else if (commandString.startsWith("stem:")) { - setStemMode(commandString.substring(5)); - } else if ("stem".equals(commandString)) { + } else if (command.startsWith("stem ")) { + setStemMode(command.substring(5)); + } else if (command.startsWith("stem:")) { + setStemMode(command.substring(5)); + } else if ("stem".equals(command)) { setStemMode(StemMode.SHORTEST); - } else if ("word".equals(commandString)) { + } else if ("word".equals(command)) { setExact(true, null); - } else if ("exact".equals(commandString)) { + } else if ("exact".equals(command)) { setExact(true, " "); - } else if ("dynteaser".equals(commandString)) { + } else if ("dynteaser".equals(command)) { setDynamicSummary(true); - } else if ("highlight".equals(commandString)) { + } else if ("highlight".equals(command)) { setHighlightSummary(true); - } else if ("lowercase".equals(commandString)) { + } else if ("lowercase".equals(command)) { setLowercase(true); - } else if (commandString.startsWith("exact ")) { - setExact(true, commandString.substring(6)); - } else if (commandString.startsWith("ngram ")) { - setNGram(true, Integer.parseInt(commandString.substring(6))); - } else if (commandString.equals("attribute")) { + } else if (command.startsWith("exact ")) { + setExact(true, command.substring(6)); + } else if (command.startsWith("ngram ")) { + setNGram(true, Integer.parseInt(command.substring(6))); + } else if (command.equals("attribute")) { setAttribute(true); - } else if (commandString.equals("default-position")) { + } else if (command.equals("default-position")) { setDefaultPosition(true); - } else if (commandString.equals("plain-tokens")) { + } else if (command.equals("plain-tokens")) { setPlainTokens(true); - } else if (commandString.equals("multivalue")) { + } else if (command.equals("multivalue")) { setMultivalue(true); - } else if (commandString.equals("fast-search")) { + } else if (command.equals("fast-search")) { setFastSearch(true); - } else if (commandString.equals("normalize")) { + } else if (command.equals("normalize")) { setNormalize(true); - } else if (commandString.equals("literal-boost")) { + } else if (command.equals("literal-boost")) { setLiteralBoost(true); - } else if (commandString.equals("numerical")) { + } else if (command.equals("numerical")) { setNumerical(true); - } else if (commandString.startsWith("predicate-bounds ")) { - setPredicateBounds(commandString.substring(17)); - } else if (commandString.equals("phrase-segmenting")) { + } else if (command.equals("predicate")) { + setPredicate(true); + } else if (command.startsWith("predicate-bounds ")) { + setPredicateBounds(command.substring(17)); + } else if (command.equals("phrase-segmenting")) { setPhraseSegmenting(true); - } else if (commandString.startsWith("phrase-segmenting ")) { - setPhraseSegmenting(Boolean.parseBoolean(commandString.substring("phrase-segmenting ".length()))); + } else if (command.startsWith("phrase-segmenting ")) { + setPhraseSegmenting(Boolean.parseBoolean(command.substring("phrase-segmenting ".length()))); } else { - commands.add(commandString); + commands.add(command); } return this; } + private void setTensor(boolean tensor) { + this.tensor = tensor; + } + + public boolean isTensor() { return tensor; } + private void setPredicateBounds(String bounds) { if ( ! bounds.startsWith("[..")) { predicateLowerBound = Long.parseLong(bounds.substring(1, bounds.indexOf(".."))); @@ -269,9 +261,7 @@ public class Index { return "(null)".equals(name); } - public boolean isAttribute() { - return isAttribute; - } + public boolean isAttribute() { return isAttribute; } public void setAttribute(boolean isAttribute) { this.isAttribute = isAttribute; @@ -306,6 +296,10 @@ public class Index { public boolean isNumerical() { return numerical; } + public void setPredicate(boolean isPredicate) { this.predicate = isPredicate; } + + public boolean isPredicate() { return predicate; } + public long getPredicateUpperBound() { return predicateUpperBound; } public long getPredicateLowerBound() { return predicateLowerBound; } diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index 76eef33d6c0..7b403ca3659 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -48,7 +48,6 @@ public class IndexFacts { static final String unionName = "unionOfAllKnown"; /** A search definition which contains the union of all settings. */ - @SuppressWarnings("deprecation") private SearchDefinition unionSearchDefinition = new SearchDefinition(unionName); private boolean frozen; @@ -334,10 +333,8 @@ public class IndexFacts { /** * Returns the index for this name. * - * @param indexName the name of the index. If this is null or empty the index - * named "default" is returned - * @return the index best matching the input parameters or the nullIndex - * (never null) if none is found + * @param indexName the name of the index. If this is null or empty the index named "default" is returned + * @return the index best matching the input parameters or the null Index (never null) if none is found */ public Index getIndex(String indexName) { return IndexFacts.this.getIndexFromDocumentTypes(indexName, documentTypes); diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java index 062a514056b..00935392683 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java @@ -109,7 +109,6 @@ public final class IndexModel { return searchDefinitions; } - @SuppressWarnings("deprecation") private SearchDefinition unionOf(Collection<SearchDefinition> searchDefinitions) { SearchDefinition union = new SearchDefinition(IndexFacts.unionName); diff --git a/container-search/src/main/java/com/yahoo/prelude/Location.java b/container-search/src/main/java/com/yahoo/prelude/Location.java index 37284bd6bcc..3e9c2382f31 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Location.java +++ b/container-search/src/main/java/com/yahoo/prelude/Location.java @@ -9,7 +9,7 @@ import java.util.StringTokenizer; /** * Location data for a geographical query. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen * @author arnej27959 */ public class Location { @@ -24,16 +24,9 @@ public class Location { private int y2 = 1; // center(x,y), radius - private int x = 1; - private int y = 1; - private int r = 1; - - // next three are now UNUSED - // ranking table, rank multiplier (scale) - // {0, 1} an int to make parsing and rendering the hit even simpler - private int tableId = 0; - private int s = 1; - private int replace = 0; + private int x = 0; + private int y = 0; + private int r = -1; private boolean renderCircle = false; private boolean renderRectangle = false; @@ -47,14 +40,14 @@ public class Location { return dimensions == l.dimensions && renderCircle == l.renderCircle && renderRectangle == l.renderRectangle - && aspect == l.aspect - && x1 == l.x1 - && x2 == l.x2 - && y1 == l.y1 - && y2 == l.y2 - && x == l.x - && y == l.y - && r == l.r; + && this.aspect == l.aspect + && this.x1 == l.x1 + && this.x2 == l.x2 + && this.y1 == l.y1 + && this.y2 == l.y2 + && this.x == l.x + && this.y == l.y + && this.r == l.r; } public boolean hasDimensions() { @@ -64,10 +57,10 @@ public class Location { if (hasDimensions() && dimensions != d) { throw new IllegalArgumentException("already has dimensions="+dimensions+", cannot change it to "+d); } - if (d == 1 || d == 2) { + if (d == 2) { dimensions = d; } else { - throw new IllegalArgumentException("Illegal location, dimensions must be 1 or 2, but was: "+d); + throw new IllegalArgumentException("Illegal location, dimensions must be 2, but was: "+d); } } public int getDimensions() { @@ -89,13 +82,13 @@ public class Location { if (px1 > px2) { throw new IllegalArgumentException("cannot have w > e"); } - x1 = px1; - x2 = px2; + this.x1 = px1; + this.x2 = px2; if (py1 > py2) { throw new IllegalArgumentException("cannot have s > n"); } - y1 = py1; - y2 = py2; + this.y1 = py1; + this.y2 = py2; renderRectangle = true; } @@ -104,12 +97,12 @@ public class Location { //no need to "optimize" for special cases, exactly 0, 30, 45, 60, or 90 degrees won't be input anyway double degrees = (double) y / 1000000d; if (degrees <= -90.0 || degrees >= +90.0) { - aspect = 0; + this.aspect = 0; return; } double radians = degrees * Math.PI / 180d; double cosLatRadians = Math.cos(radians); - aspect = (long) (cosLatRadians * 4294967295L); + this.aspect = (long) (cosLatRadians * 4294967295L); } public void setGeoCircle(double ns, double ew, double radius_in_degrees) { @@ -126,12 +119,12 @@ public class Location { if (ns < -90.1 || ns > +90.1) { throw new IllegalArgumentException("n/s location must be in range [-90,+90]"); } - if (radius_in_degrees < 0 || radius_in_degrees > 180.0) { - throw new IllegalArgumentException("radius must be in range [0,180] degrees, approximately upto 20000km"); + if (radius_in_degrees < 0) { + pr = -1; } - x = px; - y = py; - r = pr; + this.x = px; + this.y = py; + this.r = pr; renderCircle = true; adjustAspect(); } @@ -142,11 +135,11 @@ public class Location { throw new IllegalArgumentException("can only set geo circle once"); } if (radius_in_units < 0) { - throw new IllegalArgumentException("radius must be positive"); + radius_in_units = -1; } - x = px; - y = py; - r = radius_in_units; + this.x = px; + this.y = py; + this.r = radius_in_units; renderCircle = true; } @@ -158,17 +151,12 @@ public class Location { String rectPart = rectangle.substring(1,endof); StringTokenizer tokens = new StringTokenizer(rectPart, ","); setDimensions(Integer.parseInt(tokens.nextToken())); - if (dimensions == 1) { - x1 = Integer.parseInt(tokens.nextToken()); - x2 = Integer.parseInt(tokens.nextToken()); - if (tokens.hasMoreTokens()) { - throw new IllegalArgumentException("Illegal location syntax: "+rectangle); - } - } else if (dimensions == 2) { - x1 = Integer.parseInt(tokens.nextToken()); - y1 = Integer.parseInt(tokens.nextToken()); - x2 = Integer.parseInt(tokens.nextToken()); - y2 = Integer.parseInt(tokens.nextToken()); + this.x1 = Integer.parseInt(tokens.nextToken()); + this.y1 = Integer.parseInt(tokens.nextToken()); + this.x2 = Integer.parseInt(tokens.nextToken()); + this.y2 = Integer.parseInt(tokens.nextToken()); + if (tokens.hasMoreTokens()) { + throw new IllegalArgumentException("Illegal location syntax: "+rectangle); } renderRectangle = true; String theRest = rectangle.substring(endof+1).trim(); @@ -185,34 +173,24 @@ public class Location { String circlePart = circle.substring(1,endof); StringTokenizer tokens = new StringTokenizer(circlePart, ","); setDimensions(Integer.parseInt(tokens.nextToken())); - x = Integer.parseInt(tokens.nextToken()); - if (dimensions == 2) { - y = Integer.parseInt(tokens.nextToken()); - } - r = Integer.parseInt(tokens.nextToken()); + this.x = Integer.parseInt(tokens.nextToken()); + this.y = Integer.parseInt(tokens.nextToken()); + this.r = Integer.parseInt(tokens.nextToken()); Integer.parseInt(tokens.nextToken()); // was "tableId" - Integer.parseInt(tokens.nextToken()); // was "scale" (multiplier) + Integer.parseInt(tokens.nextToken()); // was "scale" Integer.parseInt(tokens.nextToken()); // was "replace" - - if (dimensions == 1) { - if (tokens.hasMoreTokens()) { - throw new IllegalArgumentException("Illegal location syntax: "+circle); - } - } - else { - if (tokens.hasMoreTokens()) { - String aspectToken = tokens.nextToken(); - if (aspectToken.equalsIgnoreCase("CalcLatLon")) { - adjustAspect(); - } else { - try { - aspect = Long.parseLong(aspectToken); - } catch (NumberFormatException nfe) { - throw new IllegalArgumentException("Aspect "+aspectToken+" for location must be an integer or 'CalcLatLon' for automatic aspect calculation.", nfe); - } - if (aspect > 4294967295L || aspect < 0) { - throw new IllegalArgumentException("Aspect "+aspect+" for location parameter must be less than 4294967296 (2^32)"); - } + if (tokens.hasMoreTokens()) { + String aspectToken = tokens.nextToken(); + if (aspectToken.equalsIgnoreCase("CalcLatLon")) { + adjustAspect(); + } else { + try { + aspect = Long.parseLong(aspectToken); + } catch (NumberFormatException nfe) { + throw new IllegalArgumentException("Aspect "+aspectToken+" for location must be an integer or 'CalcLatLon' for automatic aspect calculation.", nfe); + } + if (aspect > 4294967295L || aspect < 0) { + throw new IllegalArgumentException("Aspect "+aspect+" for location parameter must be less than 4294967296 (2^32)"); } } } @@ -248,34 +226,33 @@ public class Location { } public String toString() { + return render(false); + } + public String backendString() { + return render(true); + } + + private String render(boolean forBackend) { StringBuilder ser = new StringBuilder(); if (attribute != null) { ser.append(attribute).append(':'); } if (renderRectangle) { ser.append("[").append(dimensions).append(","); - if (dimensions == 1) { - ser.append(x1).append(","). - append(x2); - } - else { - ser.append(x1).append(","). - append(y1).append(","). - append(x2).append(","). - append(y2); - } + ser.append(x1).append(","). + append(y1).append(","). + append(x2).append(","). + append(y2); ser.append("]"); } if (renderCircle) { - ser.append("(").append(dimensions).append(",").append(x); - if (dimensions == 2) { - ser.append(",").append(y); - } - ser.append(",").append(r). - append(",").append(tableId). - append(",").append(s). - append(",").append(replace); - if (dimensions == 2 && aspect != 0) { + ser.append("(").append(dimensions).append(","). + append(this.x).append(",").append(this.y); + ser.append(",").append(forBackend ? backendRadius() : this.r). + append(",").append(0). // was "tableId" + append(",").append(1). // was "scale" + append(",").append(0); // was "replace" + if (aspect != 0) { ser.append(",").append(aspect); } ser.append(")"); @@ -289,7 +266,7 @@ public class Location { */ public int getBoundingWidth() { if (renderCircle) { - return r * 2; + return this.r * 2; } else { return x2 - x1; } @@ -301,7 +278,7 @@ public class Location { */ public int getBoundingHeight() { if (renderCircle) { - return r * 2; + return this.r * 2; } else { return y2 - y1; } @@ -358,11 +335,16 @@ public class Location { /** * Obtain circle radius (in degrees). + * Note that "no radius" or "infinite radius" is represented as -1. * May only be called when isGeoCircle() returns true. **/ public double degRadius() { checkGeoCircle(); - return 0.000001 * r; + return (this.r < 0) ? -1.0 : (0.000001 * this.r); + } + + private int backendRadius() { + return (this.r < 0) ? -1 : this.r; } /** @@ -370,7 +352,7 @@ public class Location { * For internal use. */ public int encode(ByteBuffer buffer) { - byte[] loc = Utf8.toBytes(toString()); + byte[] loc = Utf8.toBytes(backendString()); buffer.put(loc); return loc.length; } diff --git a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java index 7859a9698d9..3d1240237e4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude; -import com.yahoo.prelude.Index.Attribute; - import java.util.HashMap; import java.util.Map; @@ -17,13 +15,13 @@ import static com.yahoo.text.Lowercase.toLowerCase; // TODO: Make freezable! public class SearchDefinition { - private String name; + private final String name; /** A map of all indices in this search definition, indexed by name */ - private Map<String, Index> indices = new HashMap<>(); + private final Map<String, Index> indices = new HashMap<>(); /* A map of all indices in this search definition, indexed by lower cased name. */ - private Map<String, Index> lowerCase = new HashMap<>(); + private final Map<String, Index> lowerCase = new HashMap<>(); private String defaultPosition; diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 37b0fd7ebfb..8685400b5c1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -1,15 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.cluster; +import com.google.inject.Inject; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.container.QrConfig; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.core.documentapi.VespaDocumentAccess; import com.yahoo.container.handler.VipStatus; +import com.yahoo.documentapi.DocumentAccess; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.prelude.fastsearch.FastSearcher; import com.yahoo.prelude.fastsearch.SummaryParameters; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; @@ -61,13 +64,15 @@ public class ClusterSearcher extends Searcher { private VespaBackEndSearcher server = null; + @Inject public ClusterSearcher(ComponentId id, QrSearchersConfig qrsConfig, ClusterConfig clusterConfig, DocumentdbInfoConfig documentDbConfig, ComponentRegistry<Dispatcher> dispatchers, - FS4ResourcePool fs4ResourcePool, - VipStatus vipStatus) { + QrConfig qrConfig, + VipStatus vipStatus, + VespaDocumentAccess access) { super(id); int searchClusterIndex = clusterConfig.clusterId(); @@ -92,12 +97,12 @@ public class ClusterSearcher extends Searcher { } if (searchClusterConfig.indexingmode() == STREAMING) { - VdsStreamingSearcher searcher = vdsCluster(fs4ResourcePool.getServerId(), searchClusterIndex, - searchClusterConfig, docSumParams, documentDbConfig); + VdsStreamingSearcher searcher = vdsCluster(qrConfig.discriminator(), searchClusterIndex, + searchClusterConfig, docSumParams, documentDbConfig, access); addBackendSearcher(searcher); vipStatus.addToRotation(searcher.getName()); } else { - FastSearcher searcher = searchDispatch(searchClusterIndex, searchClusterName, fs4ResourcePool.getServerId(), + FastSearcher searcher = searchDispatch(searchClusterIndex, searchClusterName, qrConfig.discriminator(), docSumParams, documentDbConfig, dispatchers); addBackendSearcher(searcher); @@ -139,13 +144,14 @@ public class ClusterSearcher extends Searcher { int searchclusterIndex, QrSearchersConfig.Searchcluster searchClusterConfig, SummaryParameters docSumParams, - DocumentdbInfoConfig documentdbInfoConfig) { + DocumentdbInfoConfig documentdbInfoConfig, + VespaDocumentAccess access) { if (searchClusterConfig.searchdef().size() != 1) { throw new IllegalArgumentException("Search clusters in streaming search shall only contain a single searchdefinition : " + searchClusterConfig.searchdef()); } ClusterParams clusterParams = makeClusterParams(searchclusterIndex); - VdsStreamingSearcher searcher = new VdsStreamingSearcher(); - searcher.setSearchClusterConfigId(searchClusterConfig.rankprofiles().configid()); + VdsStreamingSearcher searcher = new VdsStreamingSearcher(access); + searcher.setSearchClusterName(searchClusterConfig.rankprofiles().configid()); searcher.setDocumentType(searchClusterConfig.searchdef(0)); searcher.setStorageClusterRouteSpec(searchClusterConfig.storagecluster().routespec()); searcher.init(serverId, docSumParams, clusterParams, documentdbInfoConfig); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/Base64DataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Base64DataField.java new file mode 100644 index 00000000000..d51bdc0fad1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Base64DataField.java @@ -0,0 +1,25 @@ +package com.yahoo.prelude.fastsearch; + +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.simple.Value; +import com.yahoo.prelude.hitfield.RawBase64; + +/** + * Represents a binary field that is presented as base64 + * @author baldersheim + */ +public class Base64DataField extends DocsumField { + public Base64DataField(String name) { + super(name); + } + + @Override + public String toString() { + return "field " + getName() + " type raw"; + } + + @Override + public Object convert(Inspector value) { + return new RawBase64(value.asData(Value.empty().asData())); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java index de07839e3e3..af7d98311f6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java @@ -23,7 +23,7 @@ public class DataField extends DocsumField { super(name); } - private Object convert(byte[] value) { + private RawData convert(byte[] value) { return new RawData(value); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 329a9caaf91..70ffc71495a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -2,7 +2,7 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.data.access.Inspector; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; @@ -51,6 +51,7 @@ public abstract class DocsumField { fieldFactory.put("double", DoubleField.class); fieldFactory.put("string", StringField.class); fieldFactory.put("data", DataField.class); + fieldFactory.put("raw", Base64DataField.class); fieldFactory.put("longstring", LongstringField.class); fieldFactory.put("longdata", LongdataField.class); fieldFactory.put("jsonstring", StructDataField.class); @@ -58,7 +59,7 @@ public abstract class DocsumField { fieldFactory.put("xmlstring", XMLField.class); fieldFactory.put("tensor", TensorField.class); } catch (Exception e) { - log.log(LogLevel.ERROR, "Could not initialize docsum decoding properly.", e); + log.log(Level.SEVERE, "Could not initialize docsum decoding properly.", e); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4ResourcePool.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4ResourcePool.java deleted file mode 100644 index ed9eb72d7dd..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4ResourcePool.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch; - -import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; -import com.yahoo.concurrent.ThreadFactoryFactory; -import com.yahoo.container.QrConfig; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * All users will get the same pool instance. - * - * @author baldersheim - */ -public class FS4ResourcePool extends AbstractComponent { - - private static final Logger logger = Logger.getLogger(FS4ResourcePool.class.getName()); - private static final AtomicInteger instanceCounter = new AtomicInteger(0); - private final String serverId; - private final int instanceId; - private final ExecutorService executor; - private final ScheduledExecutorService scheduledExecutor; - - @Inject - public FS4ResourcePool(QrConfig config) { - this(config.discriminator()); - } - - public FS4ResourcePool(String serverId) { - this.serverId = serverId; - instanceId = instanceCounter.getAndIncrement(); - String name = "FS4-" + instanceId; - executor = Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(name)); - scheduledExecutor = Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory(name + ".scheduled")); - } - - /** Returns an unique identifier of the server this runs in */ - public String getServerId() { return serverId; } - public ExecutorService getExecutor() { return executor; } - public ScheduledExecutorService getScheduledExecutor() { return scheduledExecutor; } - - @Override - public void deconstruct() { - logger.log(Level.INFO, "Deconstructing FS4ResourcePool with id '" + instanceId + "'."); - super.deconstruct(); - executor.shutdown(); - scheduledExecutor.shutdown(); - try { - executor.awaitTermination(10, TimeUnit.SECONDS); - scheduledExecutor.awaitTermination(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { - logger.warning("Executors failed terminating within timeout of 10 seconds : " + e); - } - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 338add37213..a639f2368a1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -42,7 +42,7 @@ public class FastHit extends Hit { private byte [] globalId; private transient byte[] sortData = null; - // TODO I supect this one can be dropped. + // TODO I suspect this one can be dropped. private transient Sorting sortDataSorting = null; /** @@ -188,9 +188,14 @@ public class FastHit extends Hit { summaries.add(0, new SummaryData(this, docsumDef, value, 1 + summaries.size())); } + /** Returns the raw summary data available in this as an unmodifiable list */ + public List<SummaryData> summaryData() { + return Collections.unmodifiableList(summaries); + } + /** * Returns values for the features listed in - * <a href="https://docs.vespa.ai/documentation/reference/search-definitions-reference.html#summary-features">summary-features</a> + * <a href="https://docs.vespa.ai/en/reference/schema-reference.html#summary-features">summary-features</a> * in the rank profile specified in the query producing this. */ public FeatureData features() { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 14604d61c0a..33ebebd8d49 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -21,8 +21,6 @@ import java.io.IOException; import java.util.Optional; import java.util.logging.Level; -import static com.yahoo.container.util.Util.quote; - /** * The searcher which forwards queries to fdispatch nodes, using the fnet/fs4 * network layer. @@ -83,7 +81,7 @@ public class FastSearcher extends VespaBackEndSearcher { @Override public Result doSearch2(Query query, Execution execution) { - if (dispatcher.searchCluster().groupSize() == 1) + if (dispatcher.searchCluster().wantedGroupSize() == 1) forceSinglePassGrouping(query); try (SearchInvoker invoker = getSearchInvoker(query)) { Result result = invoker.search(query, execution); @@ -160,7 +158,7 @@ public class FastSearcher extends VespaBackEndSearcher { } private static Optional<String> quotedSummaryClass(String summaryClass) { - return Optional.of(summaryClass == null ? "[null]" : quote(summaryClass)); + return Optional.of(summaryClass == null ? "[null]" : "'" + summaryClass + "'"); } public String toString() { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/GroupingListHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/GroupingListHit.java index 740b9592efc..2b91941d1d4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/GroupingListHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/GroupingListHit.java @@ -26,4 +26,5 @@ public class GroupingListHit extends Hit { private final List<Grouping> groupingList; private final DocsumDefinitionSet defs; + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index bc3ac6cdef1..d30e67195c3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -3,6 +3,8 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.collections.TinyIdentitySet; import com.yahoo.fs4.DocsumPacket; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NullItem; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; @@ -41,7 +43,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { private String serverId; /** The set of all document databases available in the backend handled by this searcher */ - private Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>(); + private final Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>(); private DocumentDatabase defaultDocumentDb = null; /** Default docsum class. null means "unset" and is the default value */ @@ -74,6 +76,19 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { protected abstract void doPartialFill(Result result, String summaryClass); + private boolean hasLocation(Item tree) { + if (tree instanceof GeoLocationItem) { + return true; + } + if (tree instanceof CompositeItem) { + var composite = (CompositeItem)tree; + for (Item child : composite.items()) { + if (hasLocation(child)) return true; + } + } + return false; + } + /** * Returns whether we need to send the query when fetching summaries. * This is necessary if the query requests summary features or dynamic snippeting @@ -87,6 +102,8 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { DocsumDefinition docsumDefinition = documentDb.getDocsumDefinitionSet().getDocsum(query.getPresentation().getSummary()); if (docsumDefinition.isDynamic()) return true; + if (hasLocation(query.getModel().getQueryTree())) return true; + // Needed to generate ranking features? RankProfile rankProfile = documentDb.rankProfiles().get(query.getRanking().getProfile()); if (rankProfile == null) return true; // stay safe @@ -257,7 +274,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { if (query.getRanking().getLocation() != null) { s.append(" location=") - .append(query.getRanking().getLocation().toString()); + .append(query.getRanking().getLocation().backendString()); } if (query.getGroupingSessionCache()) { diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java index 209bfd08e6b..55438aa35ba 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java @@ -21,6 +21,7 @@ import java.util.Iterator; * * @author Steinar Knutsen */ +// TODO Vespa 8: remove methods leaking org.json types (replace with Slime equivalent?) public class JSONString implements Inspectable { private Inspector value; @@ -436,6 +437,8 @@ public class JSONString implements Inspectable { return content; } + /** @deprecated Use {@link #getContent()} instead and parse content yourself */ + @Deprecated(forRemoval = true, since = "7") public Object getParsedJSON() { initContent(); if (parsedJSON == null) { @@ -444,6 +447,7 @@ public class JSONString implements Inspectable { return parsedJSON; } + @Deprecated(forRemoval = true, since = "7") public void setParsedJSON(Object parsedJSON) { this.parsedJSON = parsedJSON; } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java new file mode 100644 index 00000000000..134d0bc902a --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java @@ -0,0 +1,18 @@ +package com.yahoo.prelude.hitfield; + +import java.util.Base64; + +/** + * @author baldersheim + */ +public class RawBase64 { + private final byte[] content; + public RawBase64(byte[] content) { + this.content = content; + } + + @Override + public String toString() { + return Base64.getEncoder().encodeToString(content); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java b/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java index 27045629780..542df9d4b8b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java @@ -1,6 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query; +import com.yahoo.processing.IllegalInputException; + import java.nio.ByteBuffer; /** @@ -59,7 +61,7 @@ public class BoolItem extends TermItem { switch (stringValue.toLowerCase()) { case "true" : return true; case "false" : return false; - default: throw new IllegalArgumentException("Expected 'true' or 'false', got '" + stringValue + "'"); + default: throw new IllegalInputException("Expected 'true' or 'false', got '" + stringValue + "'"); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java index 64f759dcf9c..4609edb7446 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java @@ -46,7 +46,7 @@ public abstract class CompositeItem extends Item { Item possibleCycle = i.next(); if (this == possibleCycle) { - throw new QueryException("Cannot add " + item + " to " + this + " as it would create a cycle"); + throw new IllegalArgumentException("Cannot add " + item + " to " + this + " as it would create a cycle"); } else if (possibleCycle instanceof CompositeItem) { ensureNotInSubtree((CompositeItem) possibleCycle); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeTaggableItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeTaggableItem.java index b1912e4128d..dd015b5f040 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeTaggableItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeTaggableItem.java @@ -27,6 +27,10 @@ public abstract class CompositeTaggableItem extends CompositeItem implements Tag /** See {@link TaggableItem#setConnectivity} */ public void setConnectivity(Item item, double connectivity) { + if (!(item instanceof TaggableItem)) { + throw new IllegalArgumentException("setConnectivity item must be taggable, was: " + + item.getClass() + " [" + item + "]"); + } setHasUniqueID(true); item.setHasUniqueID(true); if (connectedItem != null) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/DotProductItem.java b/container-search/src/main/java/com/yahoo/prelude/query/DotProductItem.java index 455f81c8a90..ee41efd943c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/DotProductItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/DotProductItem.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query; +import java.util.Map; + /** * A weighted set query item to be evaluated as a sparse dot product. * @@ -11,6 +13,7 @@ package com.yahoo.prelude.query; public class DotProductItem extends WeightedSetItem { public DotProductItem(String indexName) { super(indexName); } + public DotProductItem(String indexName, Map<Object, Integer> map) { super(indexName, map); } @Override public ItemType getItemType() { return ItemType.DOTPRODUCT; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/FalseItem.java b/container-search/src/main/java/com/yahoo/prelude/query/FalseItem.java index 531a89312df..9abc6b2bdaa 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/FalseItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/FalseItem.java @@ -4,7 +4,7 @@ package com.yahoo.prelude.query; import java.nio.ByteBuffer; /** - * A query item which never matches. This is sometimes an useful output of query rewriting. + * A query item which never matches. This is sometimes a useful output of query rewriting. * * @author bratseth */ diff --git a/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java b/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java new file mode 100644 index 00000000000..ba5270e7af7 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java @@ -0,0 +1,120 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.prelude.query; + +import com.google.common.annotations.Beta; +import com.yahoo.prelude.Location; +import java.nio.ByteBuffer; + +/** + * This represents a geo-location in the query tree. + * Used for closeness(fieldname) and distance(fieldname) rank features. + * @author arnej + */ +@Beta +public class GeoLocationItem extends TermItem { + + private Location location; + + /** + * Construct from a Location, which must be geo circle with an attribute set. + **/ + public GeoLocationItem(Location location) { + this(location, location.getAttribute()); + if (! location.hasAttribute()) { + throw new IllegalArgumentException("Missing attribute on location: " + location); + } + } + + /** + * Construct from a Location and a field name. + * The Location must be a geo circle. + * If the Location has an attribute set, it must match the field name. + **/ + public GeoLocationItem(Location location, String fieldName) { + super(fieldName, false); + if (location.hasAttribute() && ! location.getAttribute().equals(fieldName)) { + throw new IllegalArgumentException("Inconsistent attribute on location: " + location.getAttribute() + + " versus fieldName: " + fieldName); + } + if (! location.isGeoCircle()) { + throw new IllegalArgumentException("GeoLocationItem only supports Geo Circles, got: " + location); + } + if (location.hasBoundingBox()) { + throw new IllegalArgumentException("GeoLocationItem does not support bounding box, got: " + location); + } + this.location = new Location(location.toString()); + this.location.setAttribute(null); // keep this in (superclass) indexName only + setNormalizable(false); + } + + public Location getLocation() { + return location; + } + + @Override + public String getRawWord() { + return stringValue(); + } + + @Override + public ItemType getItemType() { + return ItemType.GEO_LOCATION_TERM; + } + + @Override + public String getName() { + return "GEO_LOCATION"; + } + + @Override + public String stringValue() { + return location.toString(); + } + + @Override + public void setValue(String value) { + throw new UnsupportedOperationException("Cannot setValue("+value+") on "+getName()); + } + + @Override + public int hashCode() { + return java.util.Objects.hash(super.hashCode(), location); + } + + @Override + public boolean equals(Object object) { + if ( ! super.equals(object)) return false; + GeoLocationItem other = (GeoLocationItem) object; // Ensured by superclass + if ( ! location.equals(other.location)) return false; + return true; + } + + @Override + public String getIndexedString() { + return location.toString(); + } + + @Override + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); // takes care of index bytes + // TODO: use a better format for encoding the location on the wire. + putString(location.backendString(), buffer); + } + + @Override + public int getNumWords() { + return 1; + } + + @Override + public boolean isStemmed() { + return true; + } + + @Override + public boolean isWords() { + return false; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/HasIndexItem.java b/container-search/src/main/java/com/yahoo/prelude/query/HasIndexItem.java index 6641dee9780..c70ee9e4def 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/HasIndexItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/HasIndexItem.java @@ -2,8 +2,7 @@ package com.yahoo.prelude.query; /** - * An interface for items where it is useful to access an associated - * index name. + * An interface for items where it is useful to access an index name. * * @author Steinar Knutsen */ @@ -11,7 +10,7 @@ public interface HasIndexItem { String getIndexName(); - /** @return how many phrase words does this item contain */ + /** Returns how many phrase words does this item contain */ int getNumWords(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IndexedItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IndexedItem.java index 3dafa230eb8..cba289fa5d8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/IndexedItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/IndexedItem.java @@ -3,7 +3,7 @@ package com.yahoo.prelude.query; /** - * Interface for Items that is indexed + * Interface for Items that are indexed * * @author Lars Christian Jensen */ diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java index 714e8f9cb5e..1591d31f749 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java @@ -111,7 +111,7 @@ public class IntItem extends TermItem { } catch (IllegalArgumentException e) { throw new IllegalArgumentException("'" + expression + "' is not an int item expression: " + - "Expected NUMBER, '<'NUMBER, '>'NUMBER or ('['|'<')NUMBER;NUMBER(;NUMBER)?(']'|'>')", e); + "Expected NUMBER, '<'NUMBER, '>'NUMBER or ('['|'<')NUMBER;NUMBER(;NUMBER)?(']'|'>')", e); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index ea65bc7d7d2..467fbd3127c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -32,7 +32,7 @@ public abstract class Item implements Cloneable { /** * The definitions in Item.ItemType must match the ones in - * searchlib/src/searchlib/parsequery/parse.h + * searchlib/src/vespa/searchlib/parsequery/parse.h */ public static enum ItemType { OR(0), @@ -42,7 +42,7 @@ public abstract class Item implements Cloneable { WORD(4), INT(5), PHRASE(6), - PAREN(7), + PAREN(7), // TODO not used - remove on Vespa 8 PREFIX(8), SUBSTRING(9), NEAR(11), @@ -60,11 +60,12 @@ public abstract class Item implements Cloneable { PREDICATE_QUERY(23), REGEXP(24), WORD_ALTERNATIVES(25), - NEAREST_NEIGHBOR(26); + NEAREST_NEIGHBOR(26), + GEO_LOCATION_TERM(27); public final int code; - private ItemType(int code) { + ItemType(int code) { this.code = code; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/ItemHelper.java b/container-search/src/main/java/com/yahoo/prelude/query/ItemHelper.java index b535ae7b8bd..cbdaa04a49f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/ItemHelper.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/ItemHelper.java @@ -5,8 +5,9 @@ import java.util.Iterator; import java.util.List; /** - * Helper function for Item - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + * Helper functions for Item + * + * @author Arne Bergene Fossaa */ public class ItemHelper { @@ -77,5 +78,4 @@ public class ItemHelper { } } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java index 35b87ec0190..4fe977bff2b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java @@ -2,8 +2,8 @@ package com.yahoo.prelude.query; -import com.google.common.annotations.Beta; import com.yahoo.compress.IntegerCompressor; +import com.yahoo.prelude.query.textualrepresentation.Discloser; import java.nio.ByteBuffer; @@ -16,12 +16,14 @@ import java.nio.ByteBuffer; * * @author arnej */ -@Beta public class NearestNeighborItem extends SimpleTaggableItem { private int targetNumHits = 0; + private int hnswExploreAdditionalHits = 0; + private double distanceThreshold = Double.POSITIVE_INFINITY; + private boolean approximate = true; private String field; - private String queryTensorName; + private final String queryTensorName; public NearestNeighborItem(String fieldName, String queryTensorName) { this.field = fieldName; @@ -34,12 +36,30 @@ public class NearestNeighborItem extends SimpleTaggableItem { /** Returns the field name */ public String getIndexName() { return field; } + /** Returns the distance threshold for nearest-neighbor hits */ + public double getDistanceThreshold () { return this.distanceThreshold ; } + + /** Returns the number of extra hits to explore in HNSW algorithm */ + public int getHnswExploreAdditionalHits() { return hnswExploreAdditionalHits; } + + /** Returns whether approximation is allowed */ + public boolean getAllowApproximate() { return approximate; } + /** Returns the name of the query tensor */ public String getQueryTensorName() { return queryTensorName; } /** Set the K number of hits to produce */ public void setTargetNumHits(int target) { this.targetNumHits = target; } + /** Set the distance threshold for nearest-neighbor hits */ + public void setDistanceThreshold(double threshold) { this.distanceThreshold = threshold; } + + /** Set the number of extra hits to explore in HNSW algorithm */ + public void setHnswExploreAdditionalHits(int num) { this.hnswExploreAdditionalHits = num; } + + /** Set whether approximation is allowed */ + public void setAllowApproximate(boolean value) { this.approximate = value; } + @Override public void setIndexName(String index) { this.field = index; } @@ -57,7 +77,11 @@ public class NearestNeighborItem extends SimpleTaggableItem { super.encodeThis(buffer); putString(field, buffer); putString(queryTensorName, buffer); + int approxNum = (approximate ? 1 : 0); IntegerCompressor.putCompressedPositiveNumber(targetNumHits, buffer); + IntegerCompressor.putCompressedPositiveNumber(approxNum, buffer); + IntegerCompressor.putCompressedPositiveNumber(hnswExploreAdditionalHits, buffer); + buffer.putDouble(distanceThreshold); return 1; // number of encoded stack dump items } @@ -65,6 +89,21 @@ public class NearestNeighborItem extends SimpleTaggableItem { protected void appendBodyString(StringBuilder buffer) { buffer.append("{field=").append(field); buffer.append(",queryTensorName=").append(queryTensorName); - buffer.append(",targetNumHits=").append(targetNumHits).append("}"); + buffer.append(",hnsw.exploreAdditionalHits=").append(hnswExploreAdditionalHits); + buffer.append(",distanceThreshold=").append(distanceThreshold); + buffer.append(",approximate=").append(approximate); + buffer.append(",targetHits=").append(targetNumHits).append("}"); } + + @Override + public void disclose(Discloser discloser) { + super.disclose(discloser); + discloser.addProperty("field", field); + discloser.addProperty("queryTensorName", queryTensorName); + discloser.addProperty("hnsw.exploreAdditionalHits", hnswExploreAdditionalHits); + discloser.addProperty("distanceThreshold", distanceThreshold); + discloser.addProperty("approximate", approximate); + discloser.addProperty("targetHits", targetNumHits); + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java index 542f1393852..9b34fd7d62b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java @@ -19,16 +19,13 @@ public class PhraseSegmentItem extends IndexedSegmentItem { /** Whether this was explicitly written as a phrase using quotes by the user */ private boolean explicit = false; - /** - * Creates a phrase containing the same words and state (as pertinent) as - * the given SegmentAndItem. - */ - public PhraseSegmentItem(AndSegmentItem segAnd) { - super(segAnd.getRawWord(), segAnd.stringValue(), segAnd.isFromQuery(), segAnd.isStemmed(), segAnd.getOrigin()); - if (segAnd.getItemCount() > 0) { - WordItem w = (WordItem) segAnd.getItem(0); + /** Creates a phrase containing the same words and state (as pertinent) as the given SegmentAndItem. */ + public PhraseSegmentItem(AndSegmentItem andSegment) { + super(andSegment.getRawWord(), andSegment.stringValue(), andSegment.isFromQuery(), andSegment.isStemmed(), andSegment.getOrigin()); + if (andSegment.getItemCount() > 0) { + WordItem w = (WordItem) andSegment.getItem(0); setIndexName(w.getIndexName()); - for (Iterator<Item> i = segAnd.getItemIterator(); i.hasNext();) { + for (Iterator<Item> i = andSegment.getItemIterator(); i.hasNext();) { WordItem word = (WordItem) i.next(); addWordItem(word); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java index 88bae76b26d..2bf20bf7c5a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java @@ -94,7 +94,7 @@ public class QueryCanonicalizer { if (composite instanceof RankItem || composite instanceof NotItem) { collapseLevels(composite, composite.getItemIterator()); // collapse the first item only } - else if (composite instanceof AndItem || composite instanceof OrItem) { + else if (composite instanceof AndItem || composite instanceof OrItem || composite instanceof WeakAndItem) { for (ListIterator<Item> i = composite.getItemIterator(); i.hasNext(); ) collapseLevels(composite, i); } @@ -106,10 +106,17 @@ public class QueryCanonicalizer { Item child = i.next(); if (child == null) return; if (child.getClass() != composite.getClass()) return; + if (child instanceof WeakAndItem && !equalWeakAndSettings((WeakAndItem)child, (WeakAndItem)composite)) return; i.remove(); moveChildren((CompositeItem) child, i); } - + + private static boolean equalWeakAndSettings(WeakAndItem a, WeakAndItem b) { + if ( ! a.getIndexName().equals(b.getIndexName())) return false; + if (a.getN() != b.getN()) return false; + return true; + } + private static void moveChildren(CompositeItem from, ListIterator<Item> toIterator) { for (ListIterator<Item> i = from.getItemIterator(); i.hasNext(); ) toIterator.add(i.next()); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/QueryException.java b/container-search/src/main/java/com/yahoo/prelude/query/QueryException.java index 34898827c2e..7d029d69077 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/QueryException.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/QueryException.java @@ -4,10 +4,11 @@ package com.yahoo.prelude.query; /** * Runtime exception to mark errors in query parsing. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @deprecated no methods throw this */ +@Deprecated // TODO: Remove on Vespa 8 public class QueryException extends RuntimeException { - private static final long serialVersionUID = -2975856668328596533L; public QueryException(String message) { super(message); @@ -16,4 +17,5 @@ public class QueryException extends RuntimeException { public QueryException(String message, Throwable cause) { super(message, cause); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/RegExpItem.java b/container-search/src/main/java/com/yahoo/prelude/query/RegExpItem.java index 2a5c6135d71..ec3744306ed 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/RegExpItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/RegExpItem.java @@ -7,9 +7,10 @@ import java.util.regex.Pattern; /** * Match a field with the contained regular expression. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class RegExpItem extends TermItem { + private String expression; private Pattern regexp; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java index ac4e8b98b03..58bbcd7315c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java @@ -50,6 +50,7 @@ public class SameElementItem extends NonReducibleCompositeItem { super.adding(item); //TODO See if we can require only SimpleIndexedItem instead of TermItem Validator.ensureInstanceOf("Child item", item, TermItem.class); + Validator.ensureNotInstanceOf("Child item", item, WordAlternativesItem.class); TermItem asTerm = (TermItem) item; Validator.ensureNonEmpty("Struct fieldname", asTerm.getIndexName()); Validator.ensureNonEmpty("Query term", asTerm.getIndexedString()); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java index 1c3eb261f90..f70bf8021ff 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java @@ -102,7 +102,7 @@ public abstract class SegmentItem extends CompositeItem implements BlockItem { } private void dontAdd() { - throw new QueryException("Tried to add item to an immutable segment."); + throw new IllegalArgumentException("Tried to add item to an immutable segment."); } public Item removeItem(int index) { @@ -120,7 +120,7 @@ public abstract class SegmentItem extends CompositeItem implements BlockItem { } private void dontRemove() { - throw new QueryException("Tried to remove an item from an immutable segment."); + throw new IllegalArgumentException("Tried to remove an item from an immutable segment."); } // TODO: Add a getItemIterator which is safe for immutability diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SegmentingRule.java b/container-search/src/main/java/com/yahoo/prelude/query/SegmentingRule.java index 2a7089ed20e..891a7628a65 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SegmentingRule.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SegmentingRule.java @@ -7,9 +7,10 @@ package com.yahoo.prelude.query; * the default is creating a phrase, but for business reasons, some East Asian * languages use an AND instead. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @since 5.1.28 + * @author Steinar Knutsen */ public enum SegmentingRule { + LANGUAGE_DEFAULT, PHRASE, BOOLEAN_AND; + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java index 6f82f340f4b..afdc859a5b7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java @@ -27,6 +27,10 @@ public abstract class SimpleTaggableItem extends Item implements TaggableItem { @Override public void setConnectivity(Item item, double connectivity) { + if (!(item instanceof TaggableItem)) { + throw new IllegalArgumentException("setConnectivity item must be taggable, was: " + + item.getClass() + " [" + item + "]"); + } setHasUniqueID(true); item.setHasUniqueID(true); if (connectedItem != null) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SuffixItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SuffixItem.java index 7baca8d60ba..cb87ca85d9a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SuffixItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SuffixItem.java @@ -5,9 +5,8 @@ package com.yahoo.prelude.query; /** * A word that matches a suffix of words instead of a complete word. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ - public class SuffixItem extends WordItem { public SuffixItem(String suffix) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TaggableSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TaggableSegmentItem.java index 35df4bf443c..83122a0a512 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TaggableSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TaggableSegmentItem.java @@ -31,6 +31,10 @@ public abstract class TaggableSegmentItem extends SegmentItem implements Taggabl /** See {@link TaggableItem#setConnectivity} */ public void setConnectivity(Item item, double connectivity) { + if (!(item instanceof TaggableItem)) { + throw new IllegalArgumentException("setConnectivity item must be taggable, was: " + + item.getClass() + " [" + item + "]"); + } setHasUniqueID(true); item.setHasUniqueID(true); if (connectedItem != null) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java index 2c33e7a2630..5df0cd120b2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java @@ -97,37 +97,19 @@ public abstract class TermItem extends SimpleIndexedItem implements BlockItem { @Override public int getTermCount() { return 1; } - /** - * This refers to whether accent removal is a meaningful and possible - * operation for this word. It should be named "isTransformable" or similar, - * but for historical reasons that is not the case. This method has nothing - * to do with Unicode normalization. - * - * @return true if accent removal can/should be performed - */ - public boolean isNormalizable() { - return normalizable; - } + /** Returns whether accent removal is a meaningful and possible operation for this word. */ + public boolean isNormalizable() { return normalizable; } /** - * This refers to whether accent removal is a meaningful and possible - * operation for this word. It should be named "isTransformable" or similar, - * but for historical reasons that is not the case. This method has nothing - * to do with Unicode normalization. + * Sets whether accent removal is a meaningful and possible operation for this word. * * @param normalizable set to true if accent removal can/should be performed */ - public void setNormalizable(boolean normalizable) { - this.normalizable = normalizable; - } + public void setNormalizable(boolean normalizable) { this.normalizable = normalizable; } @Override - public SegmentingRule getSegmentingRule() { - return segmentingRule; - } + public SegmentingRule getSegmentingRule() { return segmentingRule; } - public void setSegmentingRule(SegmentingRule segmentingRule) { - this.segmentingRule = segmentingRule; - } + public void setSegmentingRule(SegmentingRule segmentingRule) { this.segmentingRule = segmentingRule; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java index a0467703b13..b818e916e71 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java @@ -20,6 +20,8 @@ public class TermType { public static TermType PHRASE = new TermType("phrase", PhraseItem.class, null, "\""); + public static TermType EQUIV = new TermType("equiv", EquivItem.class, null, ""); + public static TermType DEFAULT = new TermType("", CompositeItem.class, AndItem.class, ""); public final String name; @@ -57,8 +59,7 @@ public class TermType { * Returns an instance of the class corresponding to the given type, AndItem * if this is the DEFAULT type * - * @throws RuntimeException - * if an instance could not be created + * @throws RuntimeException if an instance could not be created */ public Item createItemClass() { try { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java b/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java index 72ba012ab07..5a6d4cd6382 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java @@ -18,8 +18,7 @@ public final class ToolBox { * {@link ToolBox#visit(QueryVisitor, Item)}. Return true to visit the * sub-items of the given item, return false to ignore the sub-items. * - * @param item - * each item in the query tree + * @param item each item in the query tree * @return whether or not to visit the sub-items of the argument item * (and then invoke the {@link #onExit()} method) */ diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java index a70d653b90a..c5679e113f1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java @@ -5,6 +5,7 @@ import com.yahoo.compress.IntegerCompressor; import com.yahoo.prelude.query.textualrepresentation.Discloser; import java.nio.ByteBuffer; +import java.util.Map; /** * A weighted set query item to be evaluated as a Wand with dot product scoring. @@ -17,7 +18,7 @@ import java.nio.ByteBuffer; */ public class WandItem extends WeightedSetItem { - private int targetNumHits; + private final int targetNumHits; private double scoreThreshold = 0; private double thresholdBoostFactor = 1; @@ -33,6 +34,18 @@ public class WandItem extends WeightedSetItem { } /** + * Creates an empty WandItem. + * + * @param fieldName the name of the weighted set field to search with this WandItem. + * @param targetNumHits the target for minimum number of hits to produce by the backend search operator handling this WandItem. + * @param tokens the tokens to search for + */ + public WandItem(String fieldName, int targetNumHits, Map<Object, Integer> tokens) { + super(fieldName, tokens); + this.targetNumHits = targetNumHits; + } + + /** * Sets the initial score threshold used by the backend search operator handling this WandItem. * The score of a document must be larger than this threshold in order to be considered a match. * Default value is 0.0. @@ -86,13 +99,10 @@ public class WandItem extends WeightedSetItem { protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); - buffer.append(targetNumHits); - buffer.append(","); - buffer.append(scoreThreshold); - buffer.append(","); + buffer.append(targetNumHits).append(","); + buffer.append(scoreThreshold).append(","); buffer.append(thresholdBoostFactor); - buffer.append(")"); - buffer.append(" "); + buffer.append(") "); } @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java index 033986dc90f..e8817a44133 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java @@ -18,110 +18,96 @@ import java.nio.ByteBuffer; */ public final class WeakAndItem extends NonReducibleCompositeItem { - private int N; + /** The default N used if none is specified: 100 */ + public static final int defaultN = 100; + + private int n; private String index; private int scoreThreshold = 0; - public ItemType getItemType() { - return ItemType.WEAK_AND; + /** Creates a WAND item with default N */ + public WeakAndItem() { + this(defaultN); } - public String getName() { - return "WAND"; + public WeakAndItem(int N) { + this("", N); } /** - * Make a WAND item with no children. You can mention a common index or you can mention it on each child. + * Make a WeakAnd item with no children. You can mention a common index or you can mention it on each child. * - * @param index The index it shall search. - * @param N the target for minimum number of hits to produce; + * @param index the index to search + * @param n the target for minimum number of hits to produce; * a backend will not suppress any hits in the operator * until N hits have been produced. - **/ - public WeakAndItem(String index, int N) { - this.N = N; + */ + public WeakAndItem(String index, int n) { + this.n = n; this.index = (index == null) ? "" : index; } - public WeakAndItem(int N) { - this("", N); - } - /** Sets the index name of all subitems of this */ + @Override + public ItemType getItemType() { return ItemType.WEAK_AND; } + + @Override + public String getName() { return "WEAKAND"; } + + @Override public void setIndexName(String index) { String toSet = (index == null) ? "" : index; super.setIndexName(toSet); this.index = toSet; } - public String getIndexName() { - return index; - } + public String getIndexName() { return index; } /** Appends the heading of this string - <code>[getName()]([limit]) </code> */ + @Override protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); - buffer.append(N); - buffer.append(")"); - buffer.append(" "); + buffer.append(n); + buffer.append(") "); } - /** The default N used if none is specified: 100 */ - public static final int defaultN = 100; - - /** Creates a WAND item with default N */ - public WeakAndItem() { - this(defaultN); - } - - public int getN() { - return N; - } + public int getN() { return n; } - public void setN(int N) { - this.N = N; - } + public void setN(int N) { this.n = N; } - public int getScoreThreshold() { - return scoreThreshold; - } + @Deprecated // TODO: Remove on Vespa 8 + public int getScoreThreshold() { return scoreThreshold; } /** - * Sets the score threshold used by the backend search operator handling this WeakAndItem. - * This threshold is currently only used if the WeakAndItem is searching a RISE index field. - * The score threshold then specifies the minimum dot product score a match needs to be part of the result set. - * Default value is 0. + * Noop. * - * @param scoreThreshold the score threshold. + * @deprecated has no effect */ - public void setScoreThreshold(int scoreThreshold) { - this.scoreThreshold = scoreThreshold; - } + @Deprecated // TODO: Remove on Vespa 8 + public void setScoreThreshold(int scoreThreshold) { this.scoreThreshold = scoreThreshold; } + @Override protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); - IntegerCompressor.putCompressedPositiveNumber(N, buffer); + IntegerCompressor.putCompressedPositiveNumber(n, buffer); putString(index, buffer); } @Override public void disclose(Discloser discloser) { super.disclose(discloser); - discloser.addProperty("N", N); + discloser.addProperty("N", n); } - public int hashCode() { - return super.hashCode() + 31 * N; - } + @Override + public int hashCode() { return super.hashCode() + 31 * n; } - /** - * Returns whether this item is of the same class and - * contains the same state as the given item - */ + /** Returns whether this item is of the same class and contains the same state as the given item. */ + @Override public boolean equals(Object object) { if (!super.equals(object)) return false; WeakAndItem other = (WeakAndItem) object; // Ensured by superclass - if (this.N != other.N) return false; + if (this.n != other.n) return false; return true; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java index 0e74099174f..15354f5f7d3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java @@ -25,9 +25,8 @@ import java.util.Map; */ public class WeightedSetItem extends SimpleTaggableItem { - private String indexName = ""; - - private CopyOnWriteHashMap<Object,Integer> set = new CopyOnWriteHashMap<>(1000); + private String indexName; + private CopyOnWriteHashMap<Object,Integer> set; /** Creates an empty weighted set; note you must provide an index name up front */ public WeightedSetItem(String indexName) { @@ -36,6 +35,15 @@ public class WeightedSetItem extends SimpleTaggableItem { } else { this.indexName = indexName; } + set = new CopyOnWriteHashMap<>(1000); + } + public WeightedSetItem(String indexName, Map<Object, Integer> map) { + if (indexName == null) { + this.indexName = ""; + } else { + this.indexName = indexName; + } + set = new CopyOnWriteHashMap<>(map); } public Integer addToken(long value, int weight) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java index 97c68ee3da8..d2df2aa6c89 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java @@ -12,15 +12,13 @@ import com.google.common.collect.ImmutableList; import com.yahoo.compress.IntegerCompressor; /** - * A set words with differing exactness scores to be used for literal boost - * ranking. + * A set of words with differing exactness scores to be used for literal boost ranking. * * @author Steinar Knutsen */ public class WordAlternativesItem extends TermItem { private List<Alternative> alternatives; - private int maxIndex; public static final class Alternative { @@ -49,7 +47,6 @@ public class WordAlternativesItem extends TermItem { public void setAlternatives(Collection<Alternative> terms) { this.alternatives = uniqueAlternatives(terms); - setMaxIndex(); } private static ImmutableList<Alternative> uniqueAlternatives(Collection<Alternative> terms) { @@ -68,27 +65,6 @@ public class WordAlternativesItem extends TermItem { return ImmutableList.copyOf(uniqueTerms); } - private void setMaxIndex() { - int maxIndex = 0; - int currentIndex = 0; - double maxScore = 0.0d; - boolean first = true; - for (Alternative val : this.alternatives) { - if (first) { - first = false; - maxIndex = 0; - maxScore = val.exactness; - } else { - if (val.exactness > maxScore) { - maxScore = val.exactness; - maxIndex = currentIndex; - } - } - ++currentIndex; - } - this.maxIndex = maxIndex; - } - @Override public String stringValue() { StringBuilder builder = new StringBuilder(); @@ -145,17 +121,14 @@ public class WordAlternativesItem extends TermItem { } /** - * Return an immutable snapshot of the contained terms. This list will not - * reflect later changes to the item. + * Return an immutable snapshot of the contained terms. This list will not reflect later changes to the item. * - * @return an immutable list of word alternatives and their respective - * scores + * @return an immutable list of word alternatives and their respective scores */ public List<Alternative> getAlternatives() { return alternatives; } - @Override public void encodeThis(ByteBuffer target) { super.encodeThis(target); @@ -172,17 +145,14 @@ public class WordAlternativesItem extends TermItem { * equal or higher exactness score. If the term string is present with a * lower exactness score, the new, higher score will take precedence. * - * @param term - * one of several string interpretations of the input word - * @param exactness - * how close the term string matches what the user input + * @param term one of several string interpretations of the input word + * @param exactness how close the term string matches what the user input */ public void addTerm(String term, double exactness) { // do note, Item is Cloneable, and overwriting the reference is what // saves us from overriding the method - if (alternatives.stream().anyMatch((a) -> a.word.equals(term) && a.exactness >= exactness )) { - return; - } + if (alternatives.stream().anyMatch((a) -> a.word.equals(term) && a.exactness >= exactness )) return; + List<Alternative> newTerms = new ArrayList<>(alternatives.size() + 1); newTerms.addAll(alternatives); newTerms.add(new Alternative(term, exactness)); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index cd8579be7f0..732466748eb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -19,7 +19,6 @@ import java.util.*; * @author bratseth * @author Steinar Knutsen */ -@SuppressWarnings("deprecation") public abstract class AbstractParser implements CustomParser { /** The current submodes of this parser */ @@ -48,7 +47,7 @@ public abstract class AbstractParser implements CustomParser { * of these may be active at the same time. SubModes are activated or * deactivated by specifying special indexes in the query. */ - final class Submodes { + static final class Submodes { /** * Url mode allows "_" and "-" as word characters. Default is false @@ -326,6 +325,7 @@ public abstract class AbstractParser implements CustomParser { * * @param indexName the index name which preceeded this token, or null if none * @param token the token to segment + * @param quoted whether this segment is within quoted text * @return the resulting item */ // TODO: The segmenting stuff is a mess now, this will fix it: @@ -341,7 +341,7 @@ public abstract class AbstractParser implements CustomParser { // This can be solved by making the segment method language independent by // always producing a query item containing the token text and resolve it to a WordItem or // SegmentItem after parsing and language detection. - protected Item segment(String indexName, Token token) { + protected Item segment(String indexName, Token token, boolean quoted) { String normalizedToken = normalize(token.toString()); if (token.isSpecial()) { @@ -361,12 +361,13 @@ public abstract class AbstractParser implements CustomParser { if (segments.size() == 0) { return null; } + if (segments.size() == 1) { return new WordItem(segments.get(0), "", true, token.substring); } CompositeItem composite; - if (indexFacts.getIndex(indexName).getPhraseSegmenting()) { + if (indexFacts.getIndex(indexName).getPhraseSegmenting() || quoted) { composite = new PhraseSegmentItem(token.toString(), normalizedToken, true, false, token.substring); } else { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java index e3d1b280a5a..8b878417912 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java @@ -147,11 +147,11 @@ public class AdvancedParser extends StructuredParser { return equiv; } return topLevelItem; - } else if (isTheWord("wand", item)) { - int n=consumeNumericArgument(); - if (n==0) - n=WeakAndItem.defaultN; - if (topLevelIsClosed || !(topLevelItem instanceof WeakAndItem) || n!=((WeakAndItem)topLevelItem).getN()) { + } else if (isTheWord("wand", item) || isTheWord("weakand", item)) { + int n = consumeNumericArgument(); + if (n == 0) + n = WeakAndItem.defaultN; + if (topLevelIsClosed || !(topLevelItem instanceof WeakAndItem) || n != ((WeakAndItem)topLevelItem).getN()) { WeakAndItem wand = new WeakAndItem(); wand.setN(n); wand.addItem(topLevelItem); @@ -206,7 +206,7 @@ public class AdvancedParser extends StructuredParser { if (!tokens.currentIs(LBRACE)) return 0; tokens.skip(LBRACE); if (!tokens.currentIsNoIgnore(NUMBER)) throw new IllegalArgumentException("Expected an integer argument"); - int distance=Integer.valueOf(tokens.next().image); + int distance = Integer.valueOf(tokens.next().image); if (!tokens.skip(Token.Kind.RBRACE)) throw new IllegalArgumentException("Expected a right brace following the argument"); return distance; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java index d9b969757c2..793d394801f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java @@ -30,6 +30,7 @@ public class AllParser extends SimpleParser { super(environment); } + @Override protected Item parseItems() { int position = tokens.getPosition(); try { @@ -111,6 +112,7 @@ public class AllParser extends SimpleParser { protected Item negativeItem() { int position = tokens.getPosition(); Item item = null; + boolean isComposited = false; try { if ( ! tokens.skip(MINUS)) return null; if (tokens.currentIsNoIgnore(SPACE)) return null; @@ -120,6 +122,7 @@ public class AllParser extends SimpleParser { item = compositeItem(); if (item != null) { + isComposited = true; if (item instanceof OrItem) { // Turn into And AndItem and = new AndItem(); @@ -136,9 +139,11 @@ public class AllParser extends SimpleParser { // Heuristic overdrive engaged! // Interpret -N as a positive item matching a negative number (by backtracking out of this) // but not if there is an explicit index (such as -a:b) + // but interpret -(N) as a negative item matching a positive number // but interpret --N as a negative item matching a negative number if (item instanceof IntItem && ((IntItem)item).getIndexName().isEmpty() && + ! isComposited && ! ((IntItem)item).getNumber().startsWith(("-"))) item = null; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java index dd836e9c8e1..b714a1d8b34 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java @@ -35,21 +35,12 @@ public class AnyParser extends SimpleParser { return anyItems(true); } - Item parseFilter(String filter, Language queryLanguage, Set<String> searchDefinitions) { - return parseFilter(filter, queryLanguage, environment.getIndexFacts().newSession(searchDefinitions, Collections.emptySet())); - } - Item parseFilter(String filter, Language queryLanguage, IndexFacts.Session indexFacts) { - Item filterRoot; - setState(queryLanguage, indexFacts); tokenize(filter, null, indexFacts, queryLanguage); - filterRoot = anyItems(true); - - if (filterRoot == null) { - return null; - } + Item filterRoot = anyItems(true); + if (filterRoot == null) return null; markAllTermsAsFilters(filterRoot); return filterRoot; @@ -61,18 +52,10 @@ public class AnyParser extends SimpleParser { try { tokens.skipMultiple(PLUS); + if ( ! tokens.skipMultiple(MINUS)) return null; + if (tokens.currentIsNoIgnore(SPACE)) return null; - if (!tokens.skipMultiple(MINUS)) { - return null; - } - - if (tokens.currentIsNoIgnore(SPACE)) { - return null; - } - - if (item == null) { - item = indexableItem(); - } + item = indexableItem(); if (item == null) { item = compositeItem(); @@ -88,13 +71,13 @@ public class AnyParser extends SimpleParser { } } } - if (item!=null) + if (item != null) item.setProtected(true); + return item; } finally { - if (item == null) { + if (item == null) tokens.setPosition(position); - } } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java index 6d4401aca04..12f63276269 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java @@ -23,8 +23,7 @@ public class PhraseParser extends AbstractParser { /** * Ignores everything but words and numbers * - * @return a phrase item if several words/numbers was found, - * a word item if only one was found + * @return a phrase item if several words/numbers was found, a word item if only one was found */ private Item forcedPhrase() { Item firstWord = null; @@ -38,7 +37,7 @@ public class PhraseParser extends AbstractParser { } // Note, this depends on segment never creating AndItems when quoted // (the second argument) is true. - Item newWord = segment(null, token); + Item newWord = segment(null, token, true); if (firstWord == null) { // First pass firstWord = newWord; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index 9ddfea6dffb..0686a4bdb43 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -50,32 +50,28 @@ abstract class SimpleParser extends StructuredParser { private Item anyItemsBody(boolean topLevel) { Item topLevelItem = null; NotItem not = null; - Item item; + Item item = null; do { - item = null; - - if (item == null) { - item = positiveItem(); - if (item != null) { - if (not == null) { - not = new NotItem(); - not.addPositiveItem(item); - topLevelItem = combineItems(topLevelItem, not); - } else { - not.addPositiveItem(item); - } + item = positiveItem(); + if (item != null) { + if (not == null) { + not = new NotItem(); + not.addPositiveItem(item); + topLevelItem = combineItems(topLevelItem, not); + } else { + not.addPositiveItem(item); } } if (item == null) { item = negativeItem(); if (item != null) { - if (not == null && item != null) { + if (not == null) { not = new NotItem(); not.addNegativeItem(item); topLevelItem = combineItems(topLevelItem, not); - } else if (item != null) { + } else { not.addNegativeItem(item); } } @@ -97,9 +93,8 @@ abstract class SimpleParser extends StructuredParser { if (item != null) { if (topLevelItem == null) { topLevelItem = item; - } else if (needNewTopLevel(topLevelItem, item)) { + } else if (needNewORTopLevel(topLevelItem, item)) { CompositeItem newTop = new OrItem(); - newTop.addItem(topLevelItem); newTop.addItem(item); topLevelItem = newTop; @@ -144,21 +139,13 @@ abstract class SimpleParser extends StructuredParser { } } - - /** Says whether we need a new top level item given the new item */ - private boolean needNewTopLevel(Item topLevelItem, Item item) { - if (item == null) { - return false; - } - if (topLevelItem instanceof TermItem) { - return true; - } - if (topLevelItem instanceof PhraseItem) { - return true; - } - if (topLevelItem instanceof BlockItem) { - return true; - } + /** Says whether we need a new top level OR item given the new item */ + private boolean needNewORTopLevel(Item topLevelItem, Item item) { + if (item == null) return false; + if (topLevelItem instanceof TermItem) return true; + if (topLevelItem instanceof PhraseItem) return true; + if (topLevelItem instanceof BlockItem) return true; + if ( topLevelItem instanceof AndItem) return true; return false; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java deleted file mode 100644 index 53fc552204f..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.query.parser; - -import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens; - -import java.util.*; -import java.util.logging.Logger; - - -/** - * A <i>registry</i> which is responsible for knowing the current - * set of special tokens. The default registry returns empty token lists - * for all names. Usage of this registry is multithread safe. - * - * @author bratseth - */ -public class SpecialTokenRegistry { - - /** The log of this */ - private static Logger log = Logger.getLogger(SpecialTokens.class.getName()); - - private static final SpecialTokens nullSpecialTokens = new SpecialTokens(); - - /** - * The current authorative special token lists, indexed on name. - * These lists are unmodifiable and used directly by clients of this - */ - private Map<String,SpecialTokens> specialTokenMap = new HashMap<>(); - - private boolean frozen = false; - - /** - * Creates an empty special token registry which - * does not subscribe to any configuration - */ - public SpecialTokenRegistry() {} - - /** - * Create a special token registry which subscribes to the specialtokens - * configuration. Only used for testing. - */ - public SpecialTokenRegistry(String configId) { - try { - build(new ConfigGetter<>(SpecialtokensConfig.class).getConfig(configId)); - } catch (Exception e) { - log.config( - "No special tokens are configured (" + e.getMessage() + ")"); - } - } - - /** - * Create a special token registry from a configuration object. This is the production code path. - */ - public SpecialTokenRegistry(SpecialtokensConfig config) { - if (config != null) { - build(config); - } - freeze(); - } - - private void freeze() { - frozen = true; - } - - private void build(SpecialtokensConfig config) { - List<SpecialTokens> list = new ArrayList<>(); - for (Iterator<Tokenlist> i = config.tokenlist().iterator(); i.hasNext();) { - Tokenlist tokenList = i.next(); - SpecialTokens tokens = new SpecialTokens(tokenList.name()); - - for (Iterator<Tokens> j = tokenList.tokens().iterator(); j.hasNext();) { - Tokens token = j.next(); - tokens.addSpecialToken(token.token(), token.replace()); - } - tokens.freeze(); - list.add(tokens); - } - addSpecialTokens(list); - } - - /** - * Adds a SpecialTokens instance to the registry. That is, add the - * tokens contained for the name of the SpecialTokens instance - * given. - * - * @param specialTokens the SpecialTokens object to add - */ - public void addSpecialTokens(SpecialTokens specialTokens) { - ensureNotFrozen(); - List<SpecialTokens> list = new ArrayList<>(); - list.add(specialTokens); - addSpecialTokens(list); - - } - - private void ensureNotFrozen() { - if (frozen) { - throw new IllegalStateException("Tried to modify a frozen SpecialTokenRegistry instance."); - } - } - - private void addSpecialTokens(List<SpecialTokens> list) { - HashMap<String,SpecialTokens> tokens = new HashMap<>(specialTokenMap); - for(SpecialTokens t: list) { - tokens.put(t.getName(),t); - } - specialTokenMap = tokens; - } - - - /** - * Returns the currently authorative list of special tokens for - * a given name. - * - * @param name the name of the special tokens to return - * null, the empth string or the string "default" returns - * the default ones - * @return a read-only list of SpecialToken instances, an empty list if this name - * has no special tokens - */ - public SpecialTokens getSpecialTokens(String name) { - if (name == null || name.trim().equals("")) { - name = "default"; - } - SpecialTokens specialTokens = specialTokenMap.get(name); - - if (specialTokens == null) { - return nullSpecialTokens; - } - return specialTokens; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java deleted file mode 100644 index c206ff7567e..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.query.parser; - -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.query.Substring; - -import java.util.*; -import java.util.logging.Logger; - -import static com.yahoo.language.LinguisticsCase.toLowerCase; - -/** - * A list of special tokens - string that should be threated as word - * no matter what they contain. Special tokens are case insensitive. - * - * @author bratseth - */ -public class SpecialTokens { - - private static final Logger log = Logger.getLogger(SpecialTokens.class.getName()); - - private String name; - - private List<SpecialToken> specialTokens = new ArrayList<>(); - - private boolean frozen = false; - - private int currentMaximumLength = 0; - - /** Creates a null list of special tokens */ - public SpecialTokens() { - this.name = "(null)"; - } - - public SpecialTokens(String name) { - this.name = name; - } - - /** Returns the name of this special tokens list */ - public String getName() { - return name; - } - - /** - * Adds a special token to this - * - * @param token the special token string to add - * @param replace the token to replace instances of the special token with, - * or null to keep the token - */ - public void addSpecialToken(String token, String replace) { - ensureNotFrozen(); - if (!caseIndependentLength(token)) { - return; - } - // TODO are special tokens correctly unicode normalized in reagards to query parsing? - final SpecialToken specialTokenToAdd = new SpecialToken(token, replace); - currentMaximumLength = Math.max(currentMaximumLength, specialTokenToAdd.token.length()); - specialTokens.add(specialTokenToAdd); - Collections.sort(specialTokens); - } - - private boolean caseIndependentLength(String token) { - // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust... - String asLow = toLowerCase(token); - // TODO put along with the global toLowerCase - String asHigh = token.toUpperCase(Locale.ENGLISH); - if (asLow.length() != token.length() || asHigh.length() != token.length()) { - log.log(LogLevel.ERROR, "Special token '" + token + "' has case sensitive length. Ignoring the token." - + " Please report this message in a bug to the Vespa team."); - return false; - } else { - return true; - } - } - - /** - * Returns the special token starting at the start of the given string, or null if no - * special token starts at this string - * - * @param string the string to search for a special token at the start position - * @param substring true to allow the special token to be followed by a character which does not - * mark the end of a token - */ - public SpecialToken tokenize(String string, boolean substring) { - // XXX detonator pattern token.length may be != the length of the - // matching data in string, ref caseIndependentLength(String) - final String input = toLowerCase(string.substring(0, Math.min(string.length(), currentMaximumLength))); - for (Iterator<SpecialToken> i = specialTokens.iterator(); i.hasNext();) { - SpecialTokens.SpecialToken special = i.next(); - - if (input.startsWith(special.token())) { - if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string)) - return special; - } - } - return null; - } - - private boolean tokenEndsAt(int position,String string) { - return !Character.isLetterOrDigit(string.charAt(position)); - } - - /** Returns the number of special tokens in this */ - public int size() { - return specialTokens.size(); - } - - private void ensureNotFrozen() { - if (frozen) { - throw new IllegalStateException("Tried to modify a frozen SpecialTokens instance."); - } - } - - public void freeze() { - frozen = true; - } - - /** An immutable special token */ - public final static class SpecialToken implements Comparable<SpecialToken> { - - private String token; - - private String replace; - - public SpecialToken(String token, String replace) { - this.token = toLowerCase(token); - if (replace == null || replace.trim().equals("")) { - this.replace = this.token; - } else { - this.replace = toLowerCase(replace); - } - } - - /** Returns the special token */ - public String token() { - return token; - } - - /** Returns the right replace value, never null or an empty string */ - public String replace() { - return replace; - } - - @Override - public int compareTo(SpecialToken other) { - if (this.token().length() < other.token().length()) return 1; - if (this.token().length() == other.token().length()) return 0; - return -1; - } - - @Override - public boolean equals(Object other) { - if (other == this) return true; - if ( ! (other instanceof SpecialToken)) return false; - return Objects.equals(this.token, ((SpecialToken)other).token); - } - - @Override - public int hashCode() { return token.hashCode(); } - - public Token toToken(int start,String rawSource) { - return new Token(Token.Kind.WORD, replace(), true, new Substring(start, start + token.length(), rawSource)); // XXX: Unsafe? - } - - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index 5e292a06b0f..267880fc97a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -406,13 +406,18 @@ abstract class StructuredParser extends AbstractParser { } } - /** Words for phrases also permits numerals as words */ - private Item phraseWord(String indexName, boolean insidePhrase) { + /** + * Words for phrases also permits numerals as words + * + * @param quoted whether we are consuming text within quoted + * @param insidePhrase whether we are consuming additional items for an existing phrase + */ + private Item phraseWord(String indexName, boolean quoted, boolean insidePhrase) { int position = tokens.getPosition(); Item item = null; try { - item = word(indexName); + item = word(indexName, quoted); if (item == null && tokens.currentIs(NUMBER)) { Token t = tokens.next(); @@ -434,17 +439,19 @@ abstract class StructuredParser extends AbstractParser { /** * Returns a WordItem if this is a non CJK query, - * a WordItem or PhraseSegmentItem if this is a CJK query, + * a WordItem or SegmentItem if this is a CJK query, * null if the current item is not a word + * + * @param quoted whether this token is inside quotes */ - private Item word(String indexName) { + private Item word(String indexName, boolean quoted) { int position = tokens.getPosition(); Item item = null; try { - if (!tokens.currentIs(WORD) - && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS) - && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) { + if ( ! tokens.currentIs(WORD) + && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS) + && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) { return null; } Token word = tokens.next(); @@ -452,7 +459,7 @@ abstract class StructuredParser extends AbstractParser { if (submodes.url) { item = new WordItem(word, true); } else { - item = segment(indexName, word); + item = segment(indexName, word, quoted); } if (submodes.url || submodes.site) { @@ -539,7 +546,7 @@ abstract class StructuredParser extends AbstractParser { quoted = !quoted; } - Item word = phraseWord(indexName, (firstWord != null) || (composite != null)); + Item word = phraseWord(indexName, quoted, (firstWord != null) || (composite != null)); if (word == null) { if (tokens.skipMultiple(QUOTE)) { @@ -557,6 +564,7 @@ abstract class StructuredParser extends AbstractParser { if (composite != null) { composite.addItem(word); + connectLastTermsIn(composite); } else if (firstWord != null) { if (submodes.site || submodes.url) { UriItem uriItem = new UriItem(); @@ -584,6 +592,7 @@ abstract class StructuredParser extends AbstractParser { } composite.addItem(firstWord); composite.addItem(word); + connectLastTermsIn(composite); } else if (word instanceof PhraseItem) { composite = (PhraseItem)word; } else { @@ -654,6 +663,25 @@ abstract class StructuredParser extends AbstractParser { } } + private void connectLastTermsIn(CompositeItem composite) { + int items = composite.items().size(); + if (items < 2) return; + Item nextToLast = composite.items().get(items - 2); + if (nextToLast instanceof AndSegmentItem) { + var subItems = ((AndSegmentItem) nextToLast).items(); + nextToLast = subItems.get(subItems.size() - 1); + } + if ( ! (nextToLast instanceof TermItem)) return; + Item last = composite.items().get(items - 1); + if (last instanceof AndSegmentItem) { + last = ((AndSegmentItem) last).items().get(0); + } + if (last instanceof TaggableItem) { + TermItem t1 = (TermItem) nextToLast; + t1.setConnectivity(last, 1); + } + } + private boolean addStartMarking() { if (submodes.explicitAnchoring() && tokens.currentIs(HAT)) { tokens.skip(); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java index 61f09e2f7b7..b71bd57539f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java @@ -3,11 +3,11 @@ package com.yahoo.prelude.query.parser; import com.yahoo.language.Linguistics; import com.yahoo.language.process.CharacterClasses; +import com.yahoo.language.process.SpecialTokens; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.Substring; -import java.util.Collection; import java.util.Collections; import java.util.List; @@ -20,7 +20,7 @@ import static com.yahoo.prelude.query.parser.Token.Kind.*; */ public final class Tokenizer { - private List<Token> tokens = new java.util.ArrayList<>(); + private final List<Token> tokens = new java.util.ArrayList<>(); private String source; @@ -38,7 +38,7 @@ public final class Tokenizer { /** Creates a tokenizer which initializes from a given Linguistics */ public Tokenizer(Linguistics linguistics) { - this.characterClasses=linguistics.getCharacterClasses(); + this.characterClasses = linguistics.getCharacterClasses(); } /** @@ -108,8 +108,7 @@ public final class Tokenizer { if (i >= source.length()) break; int c = source.codePointAt(i); - if (characterClasses.isLetterOrDigit(c) - || (c == '\'' && acceptApostropheAsWordCharacter(currentIndex))) { + if (characterClasses.isLetterOrDigit(c) || (c == '\'' && acceptApostropheAsWordCharacter(currentIndex))) { i = consumeWordOrNumber(i, currentIndex); } else if (Character.isWhitespace(c)) { addToken(SPACE, " ", i, i + 1); @@ -187,7 +186,6 @@ public final class Tokenizer { return true; } - @SuppressWarnings({"deprecation"}) private Index determineCurrentIndex(Index defaultIndex, IndexFacts.Session indexFacts) { int backtrack = tokens.size(); int tokencnt = 0; @@ -203,7 +201,7 @@ public final class Tokenizer { } StringBuilder tmp = new StringBuilder(); for (int i = 0; i < tokencnt; i++) { - Token useToken = tokens.get(backtrack+i); + Token useToken = tokens.get(backtrack + i); tmp.append(useToken.image); } String indexName = tmp.toString(); @@ -219,22 +217,20 @@ public final class Tokenizer { } private int consumeSpecialToken(int start) { - SpecialTokens.SpecialToken specialToken=getSpecialToken(start); - if (specialToken==null) return start; - tokens.add(specialToken.toToken(start,source)); - return start + specialToken.token().length(); + SpecialTokens.Token token = getSpecialToken(start); + if (token == null) return start; + tokens.add(toToken(token, start, source)); + return start + token.token().length(); } - private SpecialTokens.SpecialToken getSpecialToken(int start) { - if (specialTokens == null) { - return null; - } + private SpecialTokens.Token getSpecialToken(int start) { + if (specialTokens == null) return null; return specialTokens.tokenize(source.substring(start), substringSpecialTokens); } private int consumeExact(int start,Index index) { if (index.getExactTerminator() == null) return consumeHeuristicExact(start); - return consumeToTerminator(start,index.getExactTerminator()); + return consumeToTerminator(start, index.getExactTerminator()); } private boolean looksLikeExactEnd(int end) { @@ -328,7 +324,6 @@ public final class Tokenizer { wantEndQuote = true; actualStart = curPos+1; } else if (wantEndQuote && looksLikeExactEnd(curPos+1)) { - // System.err.println("seen quoted token from "+actualStart+" to "+curPos); seenSome = true; wantEndQuote = false; isQuoted = true; @@ -435,7 +430,7 @@ public final class Tokenizer { if (suffStar) { addToken(STAR, "*", starPos, starPos + 1); } - tokens.add(new Token(WORD, source.substring(actualStart, end), true, new Substring(actualStart, end, source))); // XXX: Unsafe? + tokens.add(new Token(WORD, source.substring(actualStart, end), true, new Substring(actualStart, end, source))); // skip terminating quote if (isQuoted) { @@ -451,17 +446,17 @@ public final class Tokenizer { break; end++; } - tokens.add(new Token(WORD, source.substring(start, end), true, new Substring(start, end, source))); // XXX: Unsafe start? - if (end>=source.length()) + tokens.add(new Token(WORD, source.substring(start, end), true, new Substring(start, end, source))); + if (end >= source.length()) return end; else - return end+terminator.length(); // Don't create a token for the terminator + return end + terminator.length(); // Don't create a token for the terminator } private boolean terminatorStartsAt(int start,String terminator) { - int terminatorPosition=0; - while ((terminatorPosition+start)<source.length()) { - if (source.charAt(start+terminatorPosition)!=terminator.charAt(terminatorPosition)) + int terminatorPosition = 0; + while ((terminatorPosition + start) < source.length()) { + if (source.charAt(start+terminatorPosition) != terminator.charAt(terminatorPosition)) return false; terminatorPosition++; if (terminatorPosition >= terminator.length()) @@ -473,7 +468,7 @@ public final class Tokenizer { /** Consumes a word or number <i>and/or possibly</i> a special token starting within this word or number */ private int consumeWordOrNumber(int start, Index currentIndex) { int tokenEnd = start; - SpecialTokens.SpecialToken substringSpecialToken = null; + SpecialTokens.Token substringToken = null; boolean digitsOnly = true; // int underscores = 0; // boolean underscoresOnly = true; @@ -481,8 +476,8 @@ public final class Tokenizer { while (tokenEnd < source.length()) { if (substringSpecialTokens) { - substringSpecialToken=getSpecialToken(tokenEnd); - if (substringSpecialToken!=null) break; + substringToken = getSpecialToken(tokenEnd); + if (substringToken != null) break; } int c = source.codePointAt(tokenEnd); @@ -506,7 +501,7 @@ public final class Tokenizer { // underscoresOnly = false; quotesOnly = false; } else if (c == '\'') { - if (!acceptApostropheAsWordCharacter(currentIndex)) { + if ( ! acceptApostropheAsWordCharacter(currentIndex)) { break; } // Otherwise consume apostrophes... @@ -530,19 +525,26 @@ public final class Tokenizer { } } - if (substringSpecialToken==null) + if (substringToken == null) return --tokenEnd; // TODO: test the logic around tokenEnd with friends - addToken(substringSpecialToken.toToken(tokenEnd,source)); - return --tokenEnd+substringSpecialToken.token().length(); + addToken(toToken(substringToken, tokenEnd, source)); + return --tokenEnd + substringToken.token().length(); } private void addToken(Token.Kind kind, String word, int start, int end) { - addToken(new Token(kind, word, false, new Substring(start, end, source))); // XXX: Unsafe? + addToken(new Token(kind, word, false, new Substring(start, end, source))); } private void addToken(Token token) { tokens.add(token); } + public Token toToken(SpecialTokens.Token specialToken, int start, String rawSource) { + return new Token(Token.Kind.WORD, + specialToken.replacement(), + true, + new Substring(start, start + specialToken.token().length(), rawSource)); // XXX: Unsafe? + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/CJKSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/CJKSearcher.java index ae8c289a5b0..785477d6df7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/CJKSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/CJKSearcher.java @@ -56,26 +56,31 @@ public class CJKSearcher extends Searcher { AndItem replacement = new AndItem(); for (ListIterator<Item> i = ((CompositeItem) root).getItemIterator(); i.hasNext();) { Item item = i.next(); - if (item instanceof WordItem) replacement.addItem(item); - else if (item instanceof PhraseSegmentItem) { + if (item instanceof WordItem) + replacement.addItem(item); + else if (item instanceof PhraseSegmentItem) replacement.addItem(new AndSegmentItem((PhraseSegmentItem) item)); - } - else replacement.addItem(item); // should never run, but hey... just convert and hope it's OK :) + else + replacement.addItem(item); // should never get here } return replacement; - } else if (root instanceof PhraseSegmentItem) { + } + else if (root instanceof PhraseSegmentItem) { PhraseSegmentItem asSegment = (PhraseSegmentItem) root; - if (asSegment.isExplicit() || hasOverlappingTokens(asSegment)) return root; - else return new AndSegmentItem(asSegment); - } else if (root instanceof SegmentItem) { + if (asSegment.isExplicit() || hasOverlappingTokens(asSegment)) + return root; + else + return new AndSegmentItem(asSegment); + } + else if (root instanceof SegmentItem) { return root; // avoid descending into AndSegmentItems and similar - } else if (root instanceof CompositeItem) { + } + else if (root instanceof CompositeItem) { for (ListIterator<Item> i = ((CompositeItem) root).getItemIterator(); i.hasNext();) { Item item = i.next(); Item transformedItem = transform(item); - if (item != transformedItem) { + if (item != transformedItem) i.set(transformedItem); - } } return root; } @@ -96,8 +101,7 @@ public class CJKSearcher extends Searcher { * We have overlapping tokens (see * com.yahoo.prelude.querytransform.test.CJKSearcherTestCase * .testCjkQueryWithOverlappingTokens and ParseTestCase for an explanation) - * if the sum of length of tokens is greater than the lenght of the original - * word + * if the sum of length of tokens is greater than the length of the original word */ private boolean hasOverlappingTokens(PhraseSegmentItem segments) { int segmentsLength=0; diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NoRankingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NoRankingSearcher.java index 7456f33d00f..30074f0306d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NoRankingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NoRankingSearcher.java @@ -12,8 +12,7 @@ import com.yahoo.search.query.Sorting.FieldOrder; import com.yahoo.search.searchchain.Execution; /** - * Avoid doing relevance calculations if sorting only - * on attributes. + * Avoid doing relevance calculations if sorting only on attributes. * * @author Steinar Knutsen */ diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java index f49e49c1771..be33c0ee9e1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java @@ -64,9 +64,9 @@ public class PhraseMatcher { * @throws IllegalArgumentException if FSA is null */ public PhraseMatcher(FSA phraseAutomatonFSA,boolean ignorePluralForm) { - if(phraseAutomatonFSA==null) throw new IllegalArgumentException("FSA is null"); - this.ignorePluralForm=ignorePluralForm; - phraseFSA=phraseAutomatonFSA; + if (phraseAutomatonFSA == null) throw new NullPointerException("FSA is null"); + this.ignorePluralForm = ignorePluralForm; + phraseFSA = phraseAutomatonFSA; } public boolean isEmpty() { return phraseFSA == null; } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java index 5a936d42ccc..329e886c3d3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java @@ -186,6 +186,7 @@ public class QueryRewrite { } else if ((item instanceof AndItem) || (item instanceof NearItem)) { return Recall.RECALLS_NOTHING; } else if (item instanceof RankItem) { + if (i == 0) return Recall.RECALLS_NOTHING; item.removeItem(i); } else { throw new UnsupportedOperationException(item.getClass().getName()); diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index 9a9044def2d..eb31b75cd6f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -9,7 +9,7 @@ import com.yahoo.language.Language; import com.yahoo.language.Linguistics; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.StemList; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.*; @@ -98,6 +98,7 @@ public class StemmingSearcher extends Searcher { context.language = language; context.indexFacts = indexFacts; context.reverseConnectivity = createReverseConnectivities(q.getModel().getQueryTree().getRoot()); + q.trace("Stemming with language="+language, 3); return scan(q.getModel().getQueryTree().getRoot(), context); } @@ -167,7 +168,7 @@ public class StemmingSearcher extends Searcher { if (i instanceof TermItem) { return ((TermItem) i).getOrigin(); // this should always be the case } else { - getLogger().log(LogLevel.WARNING, "Weird, BlockItem '" + b + "' was a composite containing " + + getLogger().log(Level.WARNING, "Weird, BlockItem '" + b + "' was a composite containing " + i.getClass().getName() + ", expected TermItem."); } } @@ -183,9 +184,20 @@ public class StemmingSearcher extends Searcher { Substring substring = getOffsets(current); if (segments.size() == 1) { + getLogger().log(Level.FINE, () -> "Stem '"+current.stringValue()+"' mode "+index.getStemMode() + +" and language '"+context.language+"' -> '"+segments.get(0)+"'"); TaggableItem w = singleWordSegment(current, segments.get(0), index, substring, context.insidePhrase); setMetaData(current, context.reverseConnectivity, w); return (Item) w; + } else if (getLogger().isLoggable(Level.FINE)) { + var buf = new StringBuilder(); + buf.append("Stem '").append(current.stringValue()); + buf.append("' mode ").append(index.getStemMode()); + buf.append(" and language '").append(context.language).append("' ->"); + for (StemList segment : segments) { + buf.append(" '").append(segment).append("'"); + } + getLogger().log(Level.FINE, buf.toString()); } if (context.isCJK) @@ -194,6 +206,7 @@ public class StemmingSearcher extends Searcher { composite = chooseComposite(current, ((Item) current).getParent(), indexName); for (StemList segment : segments) { + getLogger().log(Level.FINE, () -> "Stem to multiple segments '"+segment+"'"); TaggableItem w = singleWordSegment(current, segment, index, substring, context.insidePhrase); if (composite instanceof AndSegmentItem) { @@ -346,7 +359,7 @@ public class StemmingSearcher extends Searcher { default: throw new IllegalArgumentException("Unknown segmenting rule: " + current.getSegmentingRule() + ". This is a bug in Vespa, as the implementation has gotten out of sync." + - " Please create a ticket as soon as possible."); + " Please create an issue."); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java index b2f5d104890..d79386a88ed 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java @@ -78,8 +78,7 @@ public class BlendingSearcher extends Searcher { * This assumes that all hits are organized into hitgroups. If not, blending will not be performed. */ protected Result blendResults(Result result, Query q, int offset, int hits, Execution execution) { - - //Assert that there are more than one hitgroup and that there are only hitgroups on the lowest level + // Assert that there are more than one hitgroup and that there are only hitgroups on the lowest level boolean foundNonGroup = false; Iterator<Hit> hitIterator = result.hits().iterator(); List<HitGroup> groups = new ArrayList<>(); diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java index badb99c0523..709d9459e50 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java @@ -22,7 +22,6 @@ import java.util.Map; * * @author Steinar Knutsen */ -@SuppressWarnings("deprecation") @After(PhaseNames.RAW_QUERY) @Before(PhaseNames.TRANSFORMED_QUERY) public class FieldCollapsingSearcher extends Searcher { @@ -33,7 +32,7 @@ public class FieldCollapsingSearcher extends Searcher { private static final CompoundName collapseSummaryName = new CompoundName("collapse.summary"); /** Maximum number of queries to send next searcher */ - private int maxQueries = 4; + private final int maxQueries = 4; /** * The max number of hits that will be preserved per unique diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/FillSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/FillSearcher.java index 45034482bb6..5390f202ef0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/FillSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/FillSearcher.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.searcher; -import com.yahoo.component.ComponentId; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java index d11b8d86d29..4e4b4600814 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java @@ -46,14 +46,11 @@ public class JuniperSearcher extends Searcher { private static final String ELLIPSIS = "..."; - // The name of the field containing document type - private static final String MAGIC_FIELD = Hit.SDDOCNAME_FIELD; - public static final String JUNIPER_TAG_REPLACING = "JuniperTagReplacing"; - private String boldOpenTag; - private String boldCloseTag; - private String separatorTag; + private final String boldOpenTag; + private final String boldCloseTag; + private final String separatorTag; @Inject public JuniperSearcher(ComponentId id, QrSearchersConfig config) { @@ -80,13 +77,13 @@ public class JuniperSearcher extends Searcher { int worstCase = result.getHitCount(); List<Hit> hits = new ArrayList<>(worstCase); for (Iterator<Hit> i = result.hits().deepIterator(); i.hasNext();) { - Hit sniffHit = i.next(); - if ( ! (sniffHit instanceof FastHit)) continue; + Hit hit = i.next(); + if ( ! (hit instanceof FastHit)) continue; - FastHit hit = (FastHit) sniffHit; - if (hit.isFilled(summaryClass)) continue; + FastHit fastHit = (FastHit)hit; + if (fastHit.isFilled(summaryClass)) continue; - hits.add(hit); + hits.add(fastHit); } execution.fill(result, summaryClass); highlight(result.getQuery().getPresentation().getBolding(), hits.iterator(), summaryClass, @@ -102,7 +99,7 @@ public class JuniperSearcher extends Searcher { FastHit fastHit = (FastHit) hit; if (summaryClass != null && ! fastHit.isFilled(summaryClass)) continue; - Object searchDefinitionField = fastHit.getField(MAGIC_FIELD); + Object searchDefinitionField = fastHit.getField(Hit.SDDOCNAME_FIELD); if (searchDefinitionField == null) continue; for (Index index : indexFacts.getIndexes(searchDefinitionField.toString())) { diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java index 43717ecf6cd..7063a14a389 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java @@ -75,7 +75,7 @@ public class PosSearcher extends Searcher { loc.setAttribute(posAttribute); try { - if (ll == null && xy == null && bb != null) { + if (ll == null && xy == null) { parseBoundingBox(bb, loc); } else { if (ll != null && xy != null) { @@ -147,7 +147,9 @@ public class PosSearcher extends Searcher { String radius = query.properties().getString(posRadius); int radiusUnits; if (radius == null) { - radiusUnits = 5000; + double radiuskm = 50.0; + double radiusdegrees = radiuskm * km2deg; + radiusUnits = (int)(radiusdegrees * 1000000); } else if (radius.endsWith("km")) { double radiuskm = Double.valueOf(radius.substring(0, radius.length()-2)); double radiusdegrees = radiuskm * km2deg; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java index 9b6f5926b61..a8b3c76fe00 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.searcher; -import java.util.Optional; import com.yahoo.component.chain.dependencies.After; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.PredicateQueryItem; +import com.yahoo.prelude.query.SimpleIndexedItem; import com.yahoo.prelude.query.ToolBox; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -15,7 +15,8 @@ import com.yahoo.search.querytransform.BooleanSearcher; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; -import java.util.Collection; +import java.util.ArrayList; +import java.util.List; /** * Checks that predicate queries don't use values outside the defined upper/lower bounds. @@ -27,26 +28,26 @@ public class ValidatePredicateSearcher extends Searcher { @Override public Result search(Query query, Execution execution) { - Optional<ErrorMessage> e = validate(query, execution.context().getIndexFacts().newSession(query)); - if (e.isPresent()) { + List<ErrorMessage> errorMessages = validate(query, execution.context().getIndexFacts().newSession(query)); + if (!errorMessages.isEmpty()) { Result r = new Result(query); - r.hits().addError(e.get()); + errorMessages.forEach(msg -> r.hits().addError(msg)); return r; } return execution.search(query); } - private Optional<ErrorMessage> validate(Query query, IndexFacts.Session indexFacts) { + private List<ErrorMessage> validate(Query query, IndexFacts.Session indexFacts) { ValidatePredicateVisitor visitor = new ValidatePredicateVisitor(indexFacts); ToolBox.visit(visitor, query.getModel().getQueryTree().getRoot()); - return visitor.errorMessage; + return visitor.errorMessages; } private static class ValidatePredicateVisitor extends ToolBox.QueryVisitor { private final IndexFacts.Session indexFacts; - public Optional<ErrorMessage> errorMessage = Optional.empty(); + final List<ErrorMessage> errorMessages = new ArrayList<>(); public ValidatePredicateVisitor(IndexFacts.Session indexFacts) { this.indexFacts = indexFacts; @@ -57,22 +58,37 @@ public class ValidatePredicateSearcher extends Searcher { if (item instanceof PredicateQueryItem) { visit((PredicateQueryItem) item); } + if (item instanceof SimpleIndexedItem) { + visit((SimpleIndexedItem) item); + } return true; } private void visit(PredicateQueryItem item) { - Index index = getIndexFromUnionOfDocumentTypes(item); + Index index = getIndexFromUnionOfDocumentTypes(item.getIndexName()); + if (!index.isPredicate()) { + errorMessages.add(ErrorMessage.createIllegalQuery(String.format("Index '%s' is not a predicate attribute.", index.getName()))); + } for (PredicateQueryItem.RangeEntry entry : item.getRangeFeatures()) { long value = entry.getValue(); if (value < index.getPredicateLowerBound() || value > index.getPredicateUpperBound()) { - errorMessage = Optional.of(ErrorMessage.createIllegalQuery( - String.format("%s=%d outside configured predicate bounds.", entry.getKey(), value))); + errorMessages.add( + ErrorMessage.createIllegalQuery(String.format("%s=%d outside configured predicate bounds.", entry.getKey(), value))); } } } - private Index getIndexFromUnionOfDocumentTypes(PredicateQueryItem item) { - return indexFacts.getIndex(item.getIndexName()); + private void visit(SimpleIndexedItem item) { + String indexName = item.getIndexName(); + Index index = getIndexFromUnionOfDocumentTypes(indexName); + if (index.isPredicate()) { + errorMessages.add( + ErrorMessage.createIllegalQuery(String.format("Index '%s' is predicate attribute and can only be used in conjunction with a predicate query operator.", indexName))); + } + } + + private Index getIndexFromUnionOfDocumentTypes(String indexName) { + return indexFacts.getIndex(indexName); } @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java index ccc9c1d2f8f..feab8faf898 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java @@ -26,43 +26,43 @@ public class RuleBase { private String source; /** The name of the automata file used, or null if none */ - protected String automataFileName=null; + protected String automataFileName = null; /** * True if this rule base is default. * The semantics of default is left to the surrounding framework */ - private boolean isDefault=false; + private boolean isDefault = false; - private List<ProductionRule> productionRules=new java.util.ArrayList<>(); + private final List<ProductionRule> productionRules = new java.util.ArrayList<>(); - private Map<String, NamedCondition> namedConditions=new java.util.LinkedHashMap<>(); + private Map<String, NamedCondition> namedConditions = new java.util.LinkedHashMap<>(); /** The analyzer used to do evaluations over this rule base */ - private RuleEngine analyzer=new RuleEngine(this); + private final RuleEngine analyzer = new RuleEngine(this); - private static final PhraseMatcher nullPhraseMatcher=PhraseMatcher.getNullMatcher(); + private static final PhraseMatcher nullPhraseMatcher = PhraseMatcher.getNullMatcher(); /** * The matcher using an automata to match terms and phrases prior to matching rules * or the null matcher if no matcher is used. */ - private PhraseMatcher phraseMatcher=nullPhraseMatcher; + private PhraseMatcher phraseMatcher = nullPhraseMatcher; /** * The names of the rule bases included indirectly or directly in this * Ordered by first to last included */ - private Set<String> includedNames=new java.util.LinkedHashSet<>(); + private final Set<String> includedNames = new java.util.LinkedHashSet<>(); /** * True if this uses an automata, even if an automata is not present right now. Useful to validate without * having automatas available */ - private boolean usesAutomata=false; + private boolean usesAutomata = false; /** Should we allow stemmed matches? */ - private boolean stemming=true; + private boolean stemming = true; /** Creates an empty rule base. TODO: Disallow */ public RuleBase() { @@ -82,8 +82,8 @@ public class RuleBase { * @throws ParseException if the rule file can not be parsed correctly * @throws RuleBaseException if the rule file contains inconsistencies */ - public static RuleBase createFromFile(String ruleFile,String automataFile) throws java.io.IOException, ParseException { - return new RuleImporter().importFile(ruleFile,automataFile); + public static RuleBase createFromFile(String ruleFile, String automataFile) throws java.io.IOException, ParseException { + return new RuleImporter().importFile(ruleFile, automataFile); } /** @@ -96,14 +96,14 @@ public class RuleBase { * @throws com.yahoo.prelude.semantics.parser.ParseException if the rule file can not be parsed correctly * @throws com.yahoo.prelude.semantics.RuleBaseException if the rule file contains inconsistencies */ - public static RuleBase createFromString(String name,String ruleString,String automataFile) throws java.io.IOException, ParseException { - RuleBase base=new RuleImporter().importString(ruleString,automataFile,new RuleBase()); + public static RuleBase createFromString(String name, String ruleString, String automataFile) throws java.io.IOException, ParseException { + RuleBase base = new RuleImporter().importString(ruleString, automataFile, new RuleBase()); base.setName(name); return base; } /** Set to true to enable stemmed matches. True by default */ - public void setStemming(boolean stemming) { this.stemming=stemming; } + public void setStemming(boolean stemming) { this.stemming = stemming; } /** Returns whether stemmed matches are allowed. True by default */ public boolean getStemming() { return stemming; } @@ -125,19 +125,19 @@ public class RuleBase { /** Rules are order based - they are included recursively depth first */ private void inlineIncluded() { // Re-add our own conditions last to - added later overrides - Map<String, NamedCondition> thisConditions=namedConditions; - namedConditions=new LinkedHashMap<>(); + Map<String, NamedCondition> thisConditions = namedConditions; + namedConditions = new LinkedHashMap<>(); - Set<RuleBase> included=new HashSet<>(); + Set<RuleBase> included = new HashSet<>(); included.add(this); - for (ListIterator<ProductionRule> i=productionRules.listIterator(); i.hasNext(); ) { - ProductionRule rule=i.next(); + for (ListIterator<ProductionRule> i = productionRules.listIterator(); i.hasNext(); ) { + ProductionRule rule = i.next(); if ( ! (rule instanceof IncludeDirective) ) continue; i.remove(); - RuleBase toInclude=((IncludeDirective)rule).getIncludedBase(); + RuleBase toInclude = ((IncludeDirective)rule).getIncludedBase(); if ( ! included.contains(toInclude)) - toInclude.inlineIn(this,i,included); + toInclude.inlineIn(this, i, included); } namedConditions.putAll(thisConditions); @@ -147,14 +147,14 @@ public class RuleBase { * Recursively include this and everything it includes into the given rule base. * Skips bases already included in this. */ - private void inlineIn(RuleBase receiver,ListIterator<ProductionRule> receiverRules,Set<RuleBase> included) { + private void inlineIn(RuleBase receiver, ListIterator<ProductionRule> receiverRules, Set<RuleBase> included) { if (included.contains(this)) return; included.add(this); - for (Iterator<ProductionRule> i=productionRules.iterator(); i.hasNext(); ) { - ProductionRule rule=i.next(); + for (Iterator<ProductionRule> i = productionRules.iterator(); i.hasNext(); ) { + ProductionRule rule = i.next(); if (rule instanceof IncludeDirective) - ((IncludeDirective)rule).getIncludedBase().inlineIn(receiver,receiverRules,included); + ((IncludeDirective)rule).getIncludedBase().inlineIn(receiver, receiverRules, included); else receiverRules.add(rule); } @@ -164,11 +164,11 @@ public class RuleBase { /** Adds a named condition which can be referenced by rules */ public void addCondition(NamedCondition namedCondition) { - namedConditions.put(namedCondition.getName(),namedCondition); + namedConditions.put(namedCondition.getName(), namedCondition); - Condition condition=namedCondition.getCondition(); - Condition superCondition=findIncludedCondition(namedCondition.getName()); - resolveSuper(condition,superCondition); + Condition condition = namedCondition.getCondition(); + Condition superCondition = findIncludedCondition(namedCondition.getName()); + resolveSuper(condition, superCondition); } private void resolveSuper(Condition condition,Condition superCondition) { @@ -176,24 +176,22 @@ public class RuleBase { ((SuperCondition)condition).setCondition(superCondition); } else if (condition instanceof CompositeCondition) { - for (Iterator<Condition> i=((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) { - Condition subCondition=i.next(); - resolveSuper(subCondition,superCondition); + for (Iterator<Condition> i = ((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) { + Condition subCondition = i.next(); + resolveSuper(subCondition, superCondition); } } } private Condition findIncludedCondition(String name) { - for (Iterator<ProductionRule> i=productionRules.iterator(); i.hasNext(); ) { - ProductionRule rule=i.next(); + for (Iterator<ProductionRule> i = productionRules.iterator(); i.hasNext(); ) { + ProductionRule rule = i.next(); if ( ! (rule instanceof IncludeDirective) ) continue; - RuleBase included=((IncludeDirective)rule).getIncludedBase(); - NamedCondition condition=included.getCondition(name); - if (condition!=null) return condition.getCondition(); + RuleBase included = ((IncludeDirective)rule).getIncludedBase(); + NamedCondition condition = included.getCondition(name); + if (condition != null) return condition.getCondition(); included.findIncludedCondition(name); - // FIXME: dead code commented out - // if (condition!=null) return condition.getCondition(); } return null; } @@ -212,8 +210,8 @@ public class RuleBase { * change, and then re-added */ public void setName(String name) { - Validator.ensureNotNull("Rule base name",name); - this.name=name; + Validator.ensureNotNull("Rule base name", name); + this.name = name; } /** Returns the name of this rule base. This is never null. */ @@ -230,27 +228,27 @@ public class RuleBase { if ( ! new File(automataFile).exists()) throw new IllegalArgumentException("Automata file '" + automataFile + "' " + "included in " + this + " not found"); - phraseMatcher=new PhraseMatcher(automataFile); + phraseMatcher = new PhraseMatcher(automataFile); phraseMatcher.setIgnorePluralForm(true); phraseMatcher.setMatchAll(true); phraseMatcher.setMatchPhraseItems(true); phraseMatcher.setMatchSingleItems(true); setPhraseMatcher(phraseMatcher); - this.automataFileName=automataFile; + this.automataFileName = automataFile; } /** Returns the name of the automata file used, or null if none */ public String getAutomataFile() { return automataFileName; } /** Sets whether this base is default, the semantics of default is left to the application */ - public void setDefault(boolean isDefault) { this.isDefault=isDefault; } + public void setDefault(boolean isDefault) { this.isDefault = isDefault; } /** Returns whether this base is default, the semantics of default is left to the application */ public boolean isDefault() { return isDefault; } /** Thread safely sets the phrase matcher to use in this, or null to not use a phrase matcher */ public synchronized void setPhraseMatcher(PhraseMatcher matcher) { - if (matcher==null) + if (matcher == null) this.phraseMatcher = nullPhraseMatcher; else this.phraseMatcher = matcher; @@ -282,7 +280,7 @@ public class RuleBase { * Set to truew if this uses an automata, even if an automata is not present right now. * Useful to validate without having automatas available */ - void setUsesAutomata(boolean usesAutomata) { this.usesAutomata=usesAutomata; } + void setUsesAutomata(boolean usesAutomata) { this.usesAutomata = usesAutomata; } // Note that included rules are added though a list iterator, not this */ public void addRule(ProductionRule productionRule) { @@ -399,13 +397,13 @@ public class RuleBase { public String toContentString() { StringBuilder b = new StringBuilder(); for (Iterator<ProductionRule> i = productionRules.iterator(); i.hasNext(); ) { - b.append(i.next().toString()); + b.append(i.next()); b.append("\n"); } b.append("\n"); b.append("\n"); for (Iterator<NamedCondition> i = namedConditions.values().iterator(); i.hasNext(); ) { - b.append(i.next().toString()); + b.append(i.next()); b.append("\n"); } return b.toString(); @@ -414,10 +412,10 @@ public class RuleBase { /** A placeholder for an included rule base until it is inlined */ private static class IncludeDirective extends ProductionRule { - private RuleBase includedBase; + private final RuleBase includedBase; public IncludeDirective(RuleBase ruleBase) { - this.includedBase=ruleBase; + this.includedBase = ruleBase; } public RuleBase getIncludedBase() { return includedBase; } @@ -425,7 +423,6 @@ public class RuleBase { /** Not used */ public String getSymbol() { return ""; } - } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBaseException.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBaseException.java index d851c2648d5..5108777d938 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBaseException.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBaseException.java @@ -6,7 +6,6 @@ package com.yahoo.prelude.semantics; * * @author bratseth */ -@SuppressWarnings("serial") public class RuleBaseException extends RuntimeException { public RuleBaseException(String message) { @@ -14,7 +13,7 @@ public class RuleBaseException extends RuntimeException { } public RuleBaseException(String message,Exception cause) { - super(message,cause); + super(message, cause); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java index 754d14ddcb4..ac643469ab6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java @@ -28,13 +28,13 @@ public class RuleImporter { * If this is set, imported rule bases are looked up in this config * otherwise, they are looked up as files */ - private SemanticRulesConfig config = null; + private SemanticRulesConfig config; /** * Ignore requests to read automata files. * Useful to validate rule bases without having automatas present */ - private boolean ignoreAutomatas = false; + private boolean ignoreAutomatas; /** * Ignore requests to include files. @@ -61,8 +61,8 @@ public class RuleImporter { } public RuleImporter(SemanticRulesConfig config, boolean ignoreAutomatas) { - this.config=config; - this.ignoreAutomatas=ignoreAutomatas; + this.config = config; + this.ignoreAutomatas = ignoreAutomatas; } public RuleImporter(SemanticRulesConfig config, boolean ignoreAutomatas, boolean ignoreIncludes) { @@ -79,7 +79,7 @@ public class RuleImporter { * @throws ParseException if the file does not contain a valid semantic rule set */ public RuleBase importFile(String fileName) throws IOException, ParseException { - return importFile(fileName,null); + return importFile(fileName, null); } /** @@ -90,8 +90,8 @@ public class RuleImporter { * @throws java.io.IOException if the file can not be read for some reason * @throws ParseException if the file does not contain a valid semantic rule set */ - public RuleBase importFile(String fileName,String automataFile) throws IOException, ParseException { - return importFile(fileName,automataFile,null); + public RuleBase importFile(String fileName, String automataFile) throws IOException, ParseException { + return importFile(fileName, automataFile, null); } /** @@ -99,27 +99,26 @@ public class RuleImporter { * * @param fileName the rule file to use * @param automataFile the automata file to use, or null to not use any - * @param ruleBase an existing rule base to import these rules into, or null - * to create a new + * @param ruleBase an existing rule base to import these rules into, or null to create a new * @throws java.io.IOException if the file can not be read for some reason * @throws ParseException if the file does not contain a valid semantic rule set */ - public RuleBase importFile(String fileName,String automataFile,RuleBase ruleBase) throws IOException, ParseException { - ruleBase=privateImportFile(fileName,automataFile,ruleBase); + public RuleBase importFile(String fileName, String automataFile, RuleBase ruleBase) throws IOException, ParseException { + ruleBase = privateImportFile(fileName, automataFile, ruleBase); ruleBase.initialize(); return ruleBase; } - public RuleBase privateImportFile(String fileName,String automataFile,RuleBase ruleBase) throws IOException, ParseException { - BufferedReader reader=null; + public RuleBase privateImportFile(String fileName, String automataFile, RuleBase ruleBase) throws IOException, ParseException { + BufferedReader reader = null; try { - reader= IOUtils.createReader(fileName, "utf-8"); - File file=new File(fileName); - String absoluteFileName=file.getAbsolutePath(); - if (ruleBase==null) - ruleBase=new RuleBase(); + reader = IOUtils.createReader(fileName, "utf-8"); + File file = new File(fileName); + String absoluteFileName = file.getAbsolutePath(); + if (ruleBase == null) + ruleBase = new RuleBase(); ruleBase.setName(stripLastName(file.getName())); - privateImportFromReader(reader,absoluteFileName,automataFile,ruleBase); + privateImportFromReader(reader, absoluteFileName, automataFile, ruleBase); return ruleBase; } finally { @@ -129,14 +128,14 @@ public class RuleImporter { /** Imports all the rule files (files ending by "sr") in the given directory */ public List<RuleBase> importDir(String ruleBaseDir) throws IOException, ParseException { - File ruleBaseDirFile=new File(ruleBaseDir); - if (!ruleBaseDirFile.exists()) + File ruleBaseDirFile = new File(ruleBaseDir); + if ( ! ruleBaseDirFile.exists()) throw new IOException("Rule base dir '" + ruleBaseDirFile.getAbsolutePath() + "' does not exist"); - File[] files=ruleBaseDirFile.listFiles(); + File[] files = ruleBaseDirFile.listFiles(); Arrays.sort(files); - List<RuleBase> ruleBases=new java.util.ArrayList<>(); + List<RuleBase> ruleBases = new java.util.ArrayList<>(); for (File file : files) { - if (!file.getName().endsWith(".sr")) continue; + if ( ! file.getName().endsWith(".sr")) continue; RuleBase base = importFile(file.getAbsolutePath()); ruleBases.add(base); } @@ -144,50 +143,50 @@ public class RuleImporter { } /** Read and include a rule base in another */ - public void include(String ruleBaseName,RuleBase ruleBase) throws java.io.IOException, ParseException { + public void include(String ruleBaseName, RuleBase ruleBase) throws java.io.IOException, ParseException { if (ignoreIncludes) return; RuleBase include; - if (config==null) { - include=privateImportFromDirectory(ruleBaseName,ruleBase); + if (config == null) { + include = privateImportFromDirectory(ruleBaseName, ruleBase); } else { - include=privateImportFromConfig(ruleBaseName); + include = privateImportFromConfig(ruleBaseName); } ruleBase.include(include); } /** Returns an unitialized rule base */ - private RuleBase privateImportFromDirectory(String ruleBaseName,RuleBase ruleBase) throws IOException, ParseException { + private RuleBase privateImportFromDirectory(String ruleBaseName, RuleBase ruleBase) throws IOException, ParseException { RuleBase include = new RuleBase(); - String includeDir=new File(ruleBase.getSource()).getParentFile().getAbsolutePath(); + String includeDir = new File(ruleBase.getSource()).getParentFile().getAbsolutePath(); if (!ruleBaseName.endsWith(".sr")) - ruleBaseName=ruleBaseName + ".sr"; - File importFile=new File(includeDir,ruleBaseName); - if (!importFile.exists()) + ruleBaseName = ruleBaseName + ".sr"; + File importFile = new File(includeDir, ruleBaseName); + if ( ! importFile.exists()) throw new IOException("No file named '" + shortenPath(importFile.getPath()) + "'"); - return privateImportFile(importFile.getPath(),null,include); + return privateImportFile(importFile.getPath(), null, include); } /** Returns an unitialized rule base */ private RuleBase privateImportFromConfig(String ruleBaseName) throws IOException, ParseException { - SemanticRulesConfig.Rulebase ruleBaseConfig=findRuleBaseConfig(config,ruleBaseName); - if (ruleBaseConfig==null) - ruleBaseConfig=findRuleBaseConfig(config,stripLastName(ruleBaseName)); - if (ruleBaseConfig==null) + SemanticRulesConfig.Rulebase ruleBaseConfig = findRuleBaseConfig(config,ruleBaseName); + if (ruleBaseConfig == null) + ruleBaseConfig = findRuleBaseConfig(config, stripLastName(ruleBaseName)); + if (ruleBaseConfig == null) throw new ParseException("Could not find included rule base '" + ruleBaseName + "'"); return privateImportConfig(ruleBaseConfig); } - private SemanticRulesConfig.Rulebase findRuleBaseConfig(SemanticRulesConfig config,String ruleBaseName) { + private SemanticRulesConfig.Rulebase findRuleBaseConfig(SemanticRulesConfig config, String ruleBaseName) { for (Object aRulebase : config.rulebase()) { - SemanticRulesConfig.Rulebase ruleBaseConfig = (SemanticRulesConfig.Rulebase) aRulebase; + SemanticRulesConfig.Rulebase ruleBaseConfig = (SemanticRulesConfig.Rulebase)aRulebase; if (ruleBaseConfig.name().equals(ruleBaseName)) return ruleBaseConfig; } return null; } - public void setAutomata(RuleBase base,String automata) { + public void setAutomata(RuleBase base, String automata) { if (ignoreAutomatas) base.setUsesAutomata(true); // Stop it from failing on automata condition references else @@ -195,9 +194,9 @@ public class RuleImporter { } static String stripLastName(String fileName) { - int lastDotIndex=fileName.lastIndexOf("."); - if (lastDotIndex<0) return fileName; - return fileName.substring(0,lastDotIndex); + int lastDotIndex = fileName.lastIndexOf("."); + if (lastDotIndex < 0) return fileName; + return fileName.substring(0, lastDotIndex); } public RuleBase importString(String string, String automataFile) throws IOException, ParseException { @@ -217,22 +216,23 @@ public class RuleImporter { } public RuleBase importConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws IOException, ParseException { - RuleBase ruleBase=privateImportConfig(ruleBaseConfig); + RuleBase ruleBase = privateImportConfig(ruleBaseConfig); ruleBase.initialize(); return ruleBase; } /** Imports an unitialized rule base */ - public RuleBase privateImportConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws IOException, ParseException { - if (config==null) throw new IllegalStateException("Must initialize with config if importing from config"); + public RuleBase privateImportConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws ParseException { + if (config == null) throw new IllegalStateException("Must initialize with config if importing from config"); RuleBase ruleBase = new RuleBase(); ruleBase.setName(ruleBaseConfig.name()); - return privateImportFromReader(new StringReader(ruleBaseConfig.rules()),"semantic-rules.cfg", - ruleBaseConfig.automata(),ruleBase); + return privateImportFromReader(new StringReader(ruleBaseConfig.rules()), + "semantic-rules.cfg", + ruleBaseConfig.automata(),ruleBase); } - public RuleBase importFromReader(Reader reader,String sourceInfo,String automataFile) throws ParseException { - return importFromReader(reader,sourceInfo,automataFile,null); + public RuleBase importFromReader(Reader reader, String sourceInfo, String automataFile) throws ParseException { + return importFromReader(reader, sourceInfo, automataFile, null); } /** @@ -245,7 +245,7 @@ public class RuleImporter { * @throws ParseException if the reader contains illegal rule syntax */ public RuleBase importFromReader(Reader reader, String sourceName, String automataFile, RuleBase ruleBase) throws ParseException { - ruleBase=privateImportFromReader(reader, sourceName, automataFile,ruleBase); + ruleBase = privateImportFromReader(reader, sourceName, automataFile, ruleBase); ruleBase.initialize(); return ruleBase; } @@ -253,19 +253,19 @@ public class RuleImporter { /** Returns an unitialized rule base */ public RuleBase privateImportFromReader(Reader reader, String sourceName, String automataFile, RuleBase ruleBase) throws ParseException { try { - if (ruleBase==null) { - ruleBase=new RuleBase(); + if (ruleBase == null) { + ruleBase = new RuleBase(); if (sourceName == null) sourceName = "anonymous"; ruleBase.setName(sourceName); } - ruleBase.setSource(sourceName.replace('\\','/')); + ruleBase.setSource(sourceName.replace('\\', '/')); new SemanticsParser(reader).semanticRules(ruleBase, this); - if (automataFile!=null && !automataFile.isEmpty()) - ruleBase.setAutomataFile(automataFile.replace('\\','/')); + if (automataFile != null && !automataFile.isEmpty()) + ruleBase.setAutomataFile(automataFile.replace('\\', '/')); return ruleBase; } catch (Throwable t) { // also catches token mgr errors - ParseException p=new ParseException("Could not parse '" + shortenPath(sourceName) + "'"); + ParseException p = new ParseException("Could not parse '" + shortenPath(sourceName) + "'"); p.initCause(t); throw p; } @@ -277,8 +277,8 @@ public class RuleImporter { * (if rules/ is present, these rules are read from an applicatino package) */ private static String shortenPath(String path) { - int rulesIndex=path.indexOf("rules/"); - if (rulesIndex<0) return path; + int rulesIndex = path.indexOf("rules/"); + if (rulesIndex < 0) return path; return path.substring(rulesIndex); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java index dafc31c66fa..989f3040cc7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java @@ -21,7 +21,7 @@ public class Evaluation { // TODO: Retrofit query into the namespace construct private ParameterNameSpace parameterNameSpace = null; - private Query query; + private final Query query; /** The current index into the flattened item list */ private int currentIndex = 0; @@ -30,7 +30,7 @@ public class Evaluation { private List<FlattenedItem> flattenedItems; /** The rule evaluation context, can be reset once the rule is evaluated */ - private RuleEvaluation ruleEvaluation; + private final RuleEvaluation ruleEvaluation; /** * The amount of context information to collect about this evaluation. @@ -41,7 +41,7 @@ public class Evaluation { private String traceIndentation = ""; /** See RuleEngine */ - private Set<Integer> matchDigests = new HashSet<>(); + private final Set<Integer> matchDigests = new HashSet<>(); /** The previous size of this query (see RuleEngine), set on matches only */ private int previousQuerySize = 0; @@ -244,15 +244,16 @@ public class Evaluation { * @param desiredParentType the desired type of the composite which contains item when this returns */ public void insertItem(Item item, CompositeItem parent, int index, TermType desiredParentType) { - if (parent == null) { // TODO: Accommodate for termtype in this case too - query.getModel().getQueryTree().setRoot(item); - + if (isEmpty(parent)) { + CompositeItem newParent = (CompositeItem)desiredParentType.createItemClass(); + newParent.addItem(item); + query.getModel().getQueryTree().setRoot(newParent); return; } - if (parent.getItemCount()>0 && parent instanceof QueryTree && parent.getItem(0) instanceof CompositeItem) { + if (parent.getItemCount() > 0 && parent instanceof QueryTree && parent.getItem(0) instanceof CompositeItem) { // combine with the existing root instead - parent=(CompositeItem)parent.getItem(0); + parent = (CompositeItem)parent.getItem(0); if (index == 1) { // that means adding it after the existing root index = parent.getItemCount(); } @@ -260,11 +261,25 @@ public class Evaluation { if (( desiredParentType == TermType.DEFAULT || desiredParentType.hasItemClass(parent.getClass()) ) && equalIndexNameIfParentIsPhrase(item, parent)) { - addItem(parent,index,item,desiredParentType); + addItem(parent, index, item, desiredParentType); } - else { + else if (incompatible(desiredParentType, parent)) { insertIncompatibleItem(item, parent, query, desiredParentType); } + else { + insertIncompatibleItemAsParent(item, parent, query, desiredParentType); + } + } + + private boolean isEmpty(Item item) { + if (item == null) return true; + if (item instanceof QueryTree && ((QueryTree) item).isEmpty()) return true; + return false; + } + + /** Returns true if the desired type cannot have childCandidate as a child */ + private boolean incompatible(TermType desiredParentType, Item childCandidate) { + return desiredParentType == TermType.EQUIV && childCandidate.getItemType() != Item.ItemType.EQUIV; } private void addItem(CompositeItem parent, int index, Item item, TermType desiredParentType) { @@ -273,27 +288,27 @@ public class Evaluation { parent.setItem(0, item); } else if (index<=1 && !(parent.getItem(0) instanceof CompositeItem)) { // Case 2: The positive must become a composite - CompositeItem positiveComposite=(CompositeItem)desiredParentType.createItemClass(); + CompositeItem positiveComposite = (CompositeItem)desiredParentType.createItemClass(); positiveComposite.addItem(parent.getItem(0)); - positiveComposite.addItem(index,item); - parent.setItem(0,positiveComposite); + positiveComposite.addItem(index, item); + parent.setItem(0, positiveComposite); } else if (parent.getItem(0)!=null && parent.getItem(0) instanceof CompositeItem // Case 3: Add to the positive composite - && index<=((CompositeItem)parent.getItem(0)).getItemCount()) { - ((CompositeItem)parent.getItem(0)).addItem(index,item); + && index <= ((CompositeItem)parent.getItem(0)).getItemCount()) { + ((CompositeItem)parent.getItem(0)).addItem(index, item); } else { // Case 4: Add negative - parent.addItem(index,item); + parent.addItem(index, item); } } - else if (parent.getItemCount()>0 && parent instanceof QueryTree) { - CompositeItem composite=(CompositeItem)desiredParentType.createItemClass(); + else if (parent.getItemCount() > 0 && parent instanceof QueryTree) { + CompositeItem composite = (CompositeItem)desiredParentType.createItemClass(); composite.addItem(parent.getItem(0)); - composite.addItem(index,item); - parent.setItem(0,composite); + composite.addItem(index, item); + parent.setItem(0, composite); } else { - parent.addItem(index,item); + parent.addItem(index, item); } } @@ -305,32 +320,39 @@ public class Evaluation { return ((PhraseItem)parent).getIndexName().equals(((IndexedItem)item).getIndexName()); } - private void insertIncompatibleItem(Item item,CompositeItem parent,Query query,TermType desiredParentType) { + private void insertIncompatibleItem(Item item, CompositeItem parent, Query query, TermType desiredParentType) { + CompositeItem newParent; + if (desiredParentType == TermType.DEFAULT) + newParent = new AndItem(); + else + newParent = (CompositeItem)desiredParentType.createItemClass(); + + newParent.addItem(item); + parent.addItem(newParent); + } + + private void insertIncompatibleItemAsParent(Item item, CompositeItem parent, Query query, TermType desiredParentType) { // Create new parent CompositeItem newParent; - if (desiredParentType==TermType.DEFAULT) - newParent=new AndItem(); + if (desiredParentType == TermType.DEFAULT) + newParent = new AndItem(); else - newParent=(CompositeItem)desiredParentType.createItemClass(); + newParent = (CompositeItem)desiredParentType.createItemClass(); // Save previous parent parent - CompositeItem parentsParent=parent.getParent(); + CompositeItem parentsParent = parent.getParent(); // Add items to new parent newParent.addItem(parent); newParent.addItem(item); // Insert new parent as root or child of old parents parent - if (parentsParent==null) { + if (parentsParent == null) { query.getModel().getQueryTree().setRoot(newParent); } else { - int parentIndex=0; - if (parentsParent!=null) { - parentIndex=parentsParent.getItemIndex(parent); - } - parentsParent.setItem(parentIndex,newParent); + parentsParent.setItem(parentsParent.getItemIndex(parent), newParent); } } @@ -338,19 +360,19 @@ public class Evaluation { if (first instanceof NullItem) { return second; } else if (first instanceof NotItem) { - NotItem notItem=(NotItem)first; - if (termType==TermType.NOT) { + NotItem notItem = (NotItem)first; + if (termType == TermType.NOT) { notItem.addNegativeItem(second); } else { - Item newPositive=combineItems(notItem.getPositiveItem(),second,termType); + Item newPositive = combineItems(notItem.getPositiveItem(), second, termType); notItem.setPositiveItem(newPositive); } return notItem; } else if (first instanceof CompositeItem) { - CompositeItem composite=(CompositeItem)first; - CompositeItem combined=createType(termType); + CompositeItem composite = (CompositeItem)first; + CompositeItem combined = createType(termType); if (combined.getClass().equals(composite.getClass())) { composite.addItem(second); return composite; @@ -362,7 +384,7 @@ public class Evaluation { } } else if (first instanceof TermItem) { - CompositeItem combined=createType(termType); + CompositeItem combined = createType(termType); combined.addItem(first); combined.addItem(second); return combined; @@ -373,25 +395,15 @@ public class Evaluation { } private CompositeItem createType(TermType termType) { - if (termType==TermType.DEFAULT) { - if (query.getModel().getType().equals(Query.Type.ANY)) + if (termType == TermType.DEFAULT) { + if (query.getModel().getType() == Query.Type.ANY) return new OrItem(); else return new AndItem(); } - else if (termType==TermType.AND) { - return new AndItem(); - } - else if (termType==TermType.OR) { - return new OrItem(); - } - else if (termType==TermType.RANK) { - return new RankItem(); - } - else if (termType==TermType.NOT) { - return new NotItem(); + else { + return (CompositeItem)termType.createItemClass(); } - throw new IllegalArgumentException("Programing error, this method should be updated with add in RankType"); } private void flatten(Item item,int position,List<FlattenedItem> toList) { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java index 9e7c761deba..a87338879fe 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java @@ -6,6 +6,8 @@ import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.query.WordItem; +import java.util.Objects; + /** * A match * @@ -14,15 +16,15 @@ import com.yahoo.prelude.query.WordItem; public class Match { /** The start position of this match */ - private int position; + private final int position; - private TermItem item; + private final TermItem item; /** The string to replace the match by, usually item.getIndexedString() */ - private String replaceValue; + private final String replaceValue; /** The parent of the matched item */ - private CompositeItem parent=null; + private final CompositeItem parent; /** * Creates a match @@ -56,23 +58,25 @@ public class Match { */ public CompositeItem getParent() { return parent; } - public int hashCode() { - return - 17*item.getIndexedString().hashCode()+ - 33*item.getIndexName().hashCode(); - } - /** Returns a new item representing this match */ public Item toItem(String label) { - return new WordItem(getReplaceValue(),label); + var newItem = new WordItem(getReplaceValue(), label); + newItem.setWeight(item.getWeight()); + return newItem; + } + + @Override + public int hashCode() { + return Objects.hash(item.getIndexedString(), item.getIndexName()); } + @Override public boolean equals(Object o) { if (! (o instanceof Match)) return false; - Match other=(Match)o; - if (other.position!=position) return false; - if (!other.item.equals(item)) return false; + Match other = (Match)o; + if (other.position != position) return false; + if ( ! other.item.equals(item)) return false; return true; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java index 274528f1fb3..f21f861a801 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java @@ -14,12 +14,12 @@ import com.yahoo.prelude.query.PhraseItem; */ public class ReferencedMatches { - private String contextName; + private final String contextName; - private List<Match> matches=new java.util.ArrayList<>(1); + private final List<Match> matches = new java.util.ArrayList<>(1); public ReferencedMatches(String contextName) { - this.contextName=contextName; + this.contextName = contextName; } public void addMatch(Match match) { @@ -38,12 +38,12 @@ public class ReferencedMatches { * @param label the label of the matches */ public Item toItem(String label) { - if (matches.size()==0) return null; - if (matches.size()==1) return matches.get(0).toItem(label); + if (matches.size() == 0) return null; + if (matches.size() == 1) return matches.get(0).toItem(label); - PhraseItem phrase=new PhraseItem(); // TODO: Somehow allow AND items instead here + PhraseItem phrase = new PhraseItem(); // TODO: Somehow allow AND items instead here phrase.setIndexName(label); - for (Iterator<Match> i=matches.iterator(); i.hasNext(); ) { + for (Iterator<Match> i = matches.iterator(); i.hasNext(); ) { phrase.addItem(i.next().toItem(label)); } return phrase; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java index 29a781b1e68..09fcb6a424f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java @@ -10,7 +10,7 @@ import com.yahoo.prelude.semantics.rule.ProductionRule; import java.util.*; /** - * A particular evalutation of a particular rule. + * A particular evaluation of a particular rule. * * @author bratseth */ @@ -23,7 +23,7 @@ public class RuleEvaluation { // Remember that whenever state is added to this class, you // must consider whether/how to make that state backtrackable - // by savinginformation in choicepoint.state + // by saving information in choicepoint.state /** The items to match in this evaluation */ private List<FlattenedItem> items; @@ -41,12 +41,12 @@ public class RuleEvaluation { private String currentContext; /** A list of referencedMatches */ - private List<ReferencedMatches> referencedMatchesList = new java.util.ArrayList<>(); + private final List<ReferencedMatches> referencedMatchesList = new java.util.ArrayList<>(); - private List<Match> nonreferencedMatches = new java.util.ArrayList<>(); + private final List<Match> nonreferencedMatches = new java.util.ArrayList<>(); /** The evaluation owning this */ - private Evaluation evaluation; + private final Evaluation evaluation; /** The choice points saved in this evaluation */ private Stack<Choicepoint> choicepoints = null; @@ -82,7 +82,7 @@ public class RuleEvaluation { choicepoints.clear(); } - public void setMatchReferences(Set<String> matchReferences) { this.matchReferences=matchReferences; } + public void setMatchReferences(Set<String> matchReferences) { this.matchReferences = matchReferences; } /** * <p>Calculates an id which is unique for each match (the totality of the matched terms) @@ -96,23 +96,23 @@ public class RuleEvaluation { * we add other kinds of conditions.</p> */ int calculateMatchDigest(ProductionRule rule) { - int matchDigest=rule.hashCode(); - int matchCounter=1; - for (Iterator<ReferencedMatches> i=referencedMatchesList.iterator(); i.hasNext(); ) { - ReferencedMatches matches=i.next(); - int termCounter=0; - for (Iterator<Match> j=matches.matchIterator(); j.hasNext(); ) { - Match match=j.next(); - matchDigest=7*matchDigest*matchCounter+ - 71*termCounter+ - match.hashCode(); + int matchDigest = rule.hashCode(); + int matchCounter = 1; + for (Iterator<ReferencedMatches> i = referencedMatchesList.iterator(); i.hasNext(); ) { + ReferencedMatches matches = i.next(); + int termCounter = 0; + for (Iterator<Match> j = matches.matchIterator(); j.hasNext(); ) { + Match match = j.next(); + matchDigest = 7 * matchDigest * matchCounter+ + 71 * termCounter + + match.hashCode(); termCounter++; } matchCounter++; } - for (Iterator<Match> i=nonreferencedMatches.iterator(); i.hasNext(); ) { - Match match=i.next(); - matchDigest=7*matchDigest*matchCounter+match.hashCode(); + for (Iterator<Match> i = nonreferencedMatches.iterator(); i.hasNext(); ) { + Match match = i.next(); + matchDigest = 7 * matchDigest * matchCounter + match.hashCode(); matchCounter++; } return matchDigest; @@ -139,7 +139,7 @@ public class RuleEvaluation { /** Sets the current position */ public void setPosition(int position) { - this.position=position; + this.position = position; } /** Returns the total number of items to match in this evaluation */ @@ -151,21 +151,21 @@ public class RuleEvaluation { public Object getValue() { return value; } /** Sets the last value returned by a condition in this evaluatiino, or null */ - public void setValue(Object value) { this.value=value; } + public void setValue(Object value) { this.value = value; } /** Returns whether we are evaluating inside a condition which inverts the truth value */ public boolean isInNegation() { return inNegation; } /** sets whether we are evaluating inside a condition which inverts the truth value */ - public void setInNegation(boolean inNegation) { this.inNegation=inNegation; } + public void setInNegation(boolean inNegation) { this.inNegation = inNegation; } /** Returns the current position into the terms this evaluates over */ public int getPosition() { return position; } /** Sets a new current label and returns the previous one */ public String setCurrentLabel(String currentLabel) { - String oldLabel=currentLabel; - this.currentLabel=currentLabel; + String oldLabel = currentLabel; + this.currentLabel = currentLabel; return oldLabel; } @@ -179,8 +179,8 @@ public class RuleEvaluation { public FlattenedItem next() { position++; - if (position>=items.size()) { - position=items.size(); + if (position >= items.size()) { + position = items.size(); return null; } @@ -189,17 +189,16 @@ public class RuleEvaluation { // TODO: Simplistic yet. Nedd to support context nesting etc. public void entering(String context) { - if (context==null) return; - if (matchReferences!=null && matchReferences.contains(context)) - currentContext=context; - + if (context == null) return; + if (matchReferences != null && matchReferences.contains(context)) + currentContext = context; } public void leaving(String context) { - if (context==null) return; - if (currentContext==null) return; + if (context == null) return; + if (currentContext == null) return; if (currentContext.equals(context)) - currentContext=null; + currentContext = null; } /** @@ -269,7 +268,7 @@ public class RuleEvaluation { * @param termType the kind of item to index, this decides the resulting structure */ public void insertItem(Item item, CompositeItem parent, int index, TermType termType) { - evaluation.insertItem(item,parent,index,termType); + evaluation.insertItem(item, parent, index, termType); } /** Returns a read-only view of the items of this */ @@ -282,7 +281,7 @@ public class RuleEvaluation { } public void trace(int level,String string) { - evaluation.trace(level,string); + evaluation.trace(level, string); } public int getTraceLevel() { @@ -304,24 +303,24 @@ public class RuleEvaluation { * @param create true to create this choicepoint if it is missing * @return the choicepoint, or null if not present, and create is false */ - public Choicepoint getChoicepoint(Condition condition,boolean create) { - if (choicepoints==null) { - if (!create) return null; - choicepoints=new java.util.Stack<>(); + public Choicepoint getChoicepoint(Condition condition, boolean create) { + if (choicepoints == null) { + if ( ! create) return null; + choicepoints = new java.util.Stack<>(); } Choicepoint choicepoint=lookupChoicepoint(condition); - if (choicepoint==null) { - if (!create) return null; - choicepoint=new Choicepoint(this,condition); + if (choicepoint == null) { + if ( ! create) return null; + choicepoint = new Choicepoint(this, condition); choicepoints.push(choicepoint); } return choicepoint; } private Choicepoint lookupChoicepoint(Condition condition) { - for (Iterator<Choicepoint> i=choicepoints.iterator(); i.hasNext(); ) { - Choicepoint choicepoint=i.next(); - if (condition==choicepoint.getCondition()) + for (Iterator<Choicepoint> i = choicepoints.iterator(); i.hasNext(); ) { + Choicepoint choicepoint = i.next(); + if (condition == choicepoint.getCondition()) return choicepoint; } return null; @@ -337,8 +336,8 @@ public class RuleEvaluation { /** Remove all the terms recognized by this match */ public void removeMatches(ReferencedMatches matches) { - for (Iterator<Match> i=matches.matchIterator(); i.hasNext(); ) { - Match match=i.next(); + for (Iterator<Match> i = matches.matchIterator(); i.hasNext(); ) { + Match match = i.next(); removeItemByIdentity(match.getItem()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java index cad753a570b..d06c4144fbc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java @@ -19,7 +19,7 @@ import com.yahoo.protect.Validator; */ public class LiteralPhraseProduction extends TermProduction { - private List<String> terms=new ArrayList<>(); + private final List<String> terms = new ArrayList<>(); /** Creates a new produced literal phrase */ public LiteralPhraseProduction() { @@ -45,20 +45,20 @@ public class LiteralPhraseProduction extends TermProduction { public List<String> getTerms() { return Collections.unmodifiableList(terms); } public void produce(RuleEvaluation e,int offset) { - PhraseItem newPhrase=new PhraseItem(); + PhraseItem newPhrase = new PhraseItem(); newPhrase.setIndexName(getLabel()); for (String term : terms) newPhrase.addItem(new WordItem(term)); if (replacing) { - Match matched=e.getNonreferencedMatch(0); - insertMatch(e,matched,newPhrase,offset); + Match matched = e.getNonreferencedMatch(0); + insertMatch(e, matched, newPhrase, offset); } else { newPhrase.setWeight(getWeight()); - if (e.getTraceLevel()>=6) - e.trace(6,"Adding '" + newPhrase + "'"); - e.addItem(newPhrase,getTermType()); + if (e.getTraceLevel() >= 6) + e.trace(6, "Adding '" + newPhrase + "'"); + e.addItem(newPhrase, getTermType()); } } @@ -67,8 +67,8 @@ public class LiteralPhraseProduction extends TermProduction { } private String getSpaceSeparated(List<String> terms) { - StringBuilder builder=new StringBuilder(); - for (Iterator<String> i=terms.iterator(); i.hasNext(); ) { + StringBuilder builder = new StringBuilder(); + for (Iterator<String> i = terms.iterator(); i.hasNext(); ) { builder.append(i.next()); if (i.hasNext()) builder.append(" "); diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java index 69a4a87c04e..66d1df690bb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java @@ -44,31 +44,32 @@ public class LiteralTermProduction extends TermProduction { * @param literal this term word * @param termType the type of term to produce */ - public LiteralTermProduction(String label,String literal, TermType termType) { - super(label,termType); + public LiteralTermProduction(String label, String literal, TermType termType) { + super(label, termType); setLiteral(literal); } /** The literal term value, never null */ public void setLiteral(String literal) { - Validator.ensureNotNull("A produced term",literal); + Validator.ensureNotNull("A produced term", literal); this.literal=literal; } /** Returns the term word produced, never null */ public String getLiteral() { return literal; } - public void produce(RuleEvaluation e,int offset) { - WordItem newItem=new WordItem(literal,getLabel()); + public void produce(RuleEvaluation e, int offset) { + WordItem newItem = new WordItem(literal, getLabel()); if (replacing) { - Match matched=e.getNonreferencedMatch(0); - insertMatch(e,matched,newItem,offset); + Match matched = e.getNonreferencedMatch(0); + newItem.setWeight(matched.getItem().getWeight()); + insertMatch(e, matched, newItem, offset); } else { newItem.setWeight(getWeight()); - if (e.getTraceLevel()>=6) - e.trace(6,"Adding '" + newItem + "'"); - e.addItem(newItem,getTermType()); + if (e.getTraceLevel() >= 6) + e.trace(6, "Adding '" + newItem + "'"); + e.addItem(newItem, getTermType()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java index 3073e2f7727..716622a4849 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java @@ -33,7 +33,7 @@ public abstract class Production { public void setPosition(int position) { this.position = position; } /** Sets the weight of this production as a percentage (default is 100) */ - public void setWeight(int weight) { this.weight=weight; } + public void setWeight(int weight) { this.weight = weight; } /** Returns the weight of this production as a percentage (default is 100) */ public int getWeight() { return weight; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java index 5b973228b1f..49d15710cd7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java @@ -15,10 +15,10 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; */ public class ProductionList { - private List<Production> productions =new java.util.ArrayList<>(); + private final List<Production> productions = new java.util.ArrayList<>(); /** True to replace by the production, false to add it */ - private boolean replacing=true; + private boolean replacing = true; public void addProduction(Production term) { term.setReplacing(replacing); @@ -28,12 +28,12 @@ public class ProductionList { /** True to replace, false to add, default true */ void setReplacing(boolean replacing) { - for (Iterator<Production> i=productions.iterator(); i.hasNext(); ) { - Production production=i.next(); + for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) { + Production production = i.next(); production.setReplacing(replacing); } - this.replacing=replacing; + this.replacing = replacing; } /** Returns an unmodifiable view of the productions in this */ @@ -42,22 +42,22 @@ public class ProductionList { public int getTermCount() { return productions.size(); } void addMatchReferences(Set<String> matchReferences) { - for (Iterator<Production> i=productions.iterator(); i.hasNext(); ) { - Production term=i.next(); + for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) { + Production term = i.next(); term.addMatchReferences(matchReferences); } } public void produce(RuleEvaluation e) { - for (int i=0; i<productions.size(); i++) { - productions.get(i).produce(e,i); + for (int i = 0; i < productions.size(); i++) { + productions.get(i).produce(e, i); } } public String toString() { - StringBuilder buffer=new StringBuilder(); - for (Iterator<Production> i=productions.iterator(); i.hasNext(); ) { - buffer.append(i.next().toString()); + StringBuilder buffer = new StringBuilder(); + for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) { + buffer.append(i.next()); if (i.hasNext()) buffer.append(" "); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java index d4593a5dbb9..e1bb8ff102d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java @@ -48,7 +48,7 @@ public class ReferenceTermProduction extends TermProduction { * @param label the label of the produced term * @param reference the label of the condition this should take it's value from */ - public ReferenceTermProduction(String label,String reference) { + public ReferenceTermProduction(String label, String reference) { super(label); setReference(reference); } @@ -60,15 +60,15 @@ public class ReferenceTermProduction extends TermProduction { * @param reference the label of the condition this should take it's value from * @param termType the type of term to produce */ - public ReferenceTermProduction(String label,String reference, TermType termType) { - super(label,termType); + public ReferenceTermProduction(String label, String reference, TermType termType) { + super(label, termType); setReference(reference); } /** The label of the condition this should take its value from, never null */ public void setReference(String reference) { Validator.ensureNotNull("reference name of a produced reference term",reference); - this.reference =reference; + this.reference = reference; } /** Returns the label of the condition this should take its value from, never null */ @@ -78,29 +78,29 @@ public class ReferenceTermProduction extends TermProduction { matchReferences.add(reference); } - public void produce(RuleEvaluation e,int offset) { - ReferencedMatches referencedMatches=e.getReferencedMatches(reference); - if (referencedMatches==null) + public void produce(RuleEvaluation e, int offset) { + ReferencedMatches referencedMatches = e.getReferencedMatches(reference); + if (referencedMatches == null) throw new EvaluationException("Referred match '" + reference + "' not found"); if (replacing) { - replaceMatches(e,referencedMatches); + replaceMatches(e, referencedMatches); } else { - addMatches(e,referencedMatches); + addMatches(e, referencedMatches); } } - public void replaceMatches(RuleEvaluation e,ReferencedMatches referencedMatches) { - Item referencedItem=referencedMatches.toItem(getLabel()); - if (referencedItem==null) return; + public void replaceMatches(RuleEvaluation e, ReferencedMatches referencedMatches) { + Item referencedItem = referencedMatches.toItem(getLabel()); + if (referencedItem == null) return; e.removeMatches(referencedMatches); - insertMatch(e, referencedMatches.matchIterator().next(),referencedItem,0); + insertMatch(e, referencedMatches.matchIterator().next(), referencedItem, 0); } - private void addMatches(RuleEvaluation e,ReferencedMatches referencedMatches) { - Item referencedItem=referencedMatches.toItem(getLabel()); - if (referencedItem==null) return; - e.addItem(referencedItem,getTermType()); + private void addMatches(RuleEvaluation e, ReferencedMatches referencedMatches) { + Item referencedItem = referencedMatches.toItem(getLabel()); + if (referencedItem == null) return; + e.addItem(referencedItem, getTermType()); } public String toInnerTermString() { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java index dfa423ec889..70151ea479b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java @@ -14,7 +14,7 @@ public class ReplacingProductionRule extends ProductionRule { /** Carries out the production of this rule */ public void produce(RuleEvaluation e) { removeNonreferencedMatches(e); - if (e.getTraceLevel()>=5) { + if (e.getTraceLevel() >= 5) { e.trace(5,"Removed terms to get '" + e.getEvaluation().getQuery().getModel().getQueryTree().getRoot() + "', will add terms"); } super.produce(e); @@ -22,16 +22,16 @@ public class ReplacingProductionRule extends ProductionRule { /** Remove items until there's only one item left */ private void removeNonreferencedMatches(RuleEvaluation e) { - int itemCount=e.getEvaluation().getQuerySize(); + int itemCount = e.getEvaluation().getQuerySize(); // Remove items backwards to ease index handling - for (int i=e.getNonreferencedMatchCount()-1; i>=0; i--) { + for (int i = e.getNonreferencedMatchCount() - 1; i >= 0; i--) { // Ensure we don't produce an empty query - if (getProduction().getTermCount()==0 && itemCount==1) + if (getProduction().getTermCount() == 0 && itemCount == 1) break; itemCount--; - Match match=e.getNonreferencedMatch(i); + Match match = e.getNonreferencedMatch(i); match.getItem().getParent().removeItem(match.getPosition()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java index 847014ff646..d1a74a991da 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java @@ -51,7 +51,7 @@ public abstract class TermProduction extends Production { /** Sets the term type to produce */ public void setTermType(TermType termType) { - Validator.ensureNotNull("Type of produced Term",termType); + Validator.ensureNotNull("Type of produced Term", termType); this.termType = termType; } @@ -60,19 +60,21 @@ public abstract class TermProduction extends Production { * TODO: Move to ruleevaluation */ protected void insertMatch(RuleEvaluation e, Match matched, Item newItem, int offset) { - newItem.setWeight(getWeight()); - int insertPosition=matched.getPosition()+offset; + if (getWeight() != 100) + newItem.setWeight(getWeight()); + int insertPosition = matched.getPosition()+offset; // This check is necessary (?) because earlier items may have been removed // after we recorded the match position. It is sort of hackish. A cleaner // solution would be to update the match position on changes - if (insertPosition>matched.getParent().getItemCount()) { - insertPosition=matched.getParent().getItemCount(); + if (insertPosition > matched.getParent().getItemCount()) { + insertPosition = matched.getParent().getItemCount(); } - e.insertItem(newItem,matched.getParent(),insertPosition,getTermType()); - if (e.getTraceLevel()>=6) - e.trace(6,"Inserted item '" + newItem + "' at position " + insertPosition + " producing " + e.getEvaluation().getQuery().getModel().getQueryTree()); + e.insertItem(newItem, matched.getParent(), insertPosition,getTermType()); + if (e.getTraceLevel() >= 6) + e.trace(6, "Inserted item '" + newItem + "' at position " + insertPosition + " producing " + + e.getEvaluation().getQuery().getModel().getQueryTree()); } protected String getLabelString() { diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index ebea2f95af7..24b631f1773 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -5,9 +5,9 @@ import com.yahoo.component.chain.dependencies.Before; import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.container.protect.Error; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; -import com.yahoo.metrics.simple.MetricSettings; +import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.metrics.simple.MetricSettings; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -29,7 +29,18 @@ import java.util.PriorityQueue; import java.util.Queue; import java.util.logging.Level; -import static com.yahoo.container.protect.Error.*; +import static com.yahoo.container.protect.Error.BACKEND_COMMUNICATION_ERROR; +import static com.yahoo.container.protect.Error.EMPTY_DOCUMENTS; +import static com.yahoo.container.protect.Error.ERROR_IN_PLUGIN; +import static com.yahoo.container.protect.Error.ILLEGAL_QUERY; +import static com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR; +import static com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER; +import static com.yahoo.container.protect.Error.INVALID_QUERY_TRANSFORMATION; +import static com.yahoo.container.protect.Error.NO_BACKENDS_IN_SERVICE; +import static com.yahoo.container.protect.Error.RESULT_HAS_ERRORS; +import static com.yahoo.container.protect.Error.SERVER_IS_MISCONFIGURED; +import static com.yahoo.container.protect.Error.TIMEOUT; +import static com.yahoo.container.protect.Error.UNSPECIFIED; /** @@ -55,39 +66,40 @@ public class StatisticsSearcher extends Searcher { private static final String FAILED_QUERIES_METRIC = "failed_queries"; private static final String MEAN_QUERY_LATENCY_METRIC = "mean_query_latency"; private static final String QUERY_LATENCY_METRIC = "query_latency"; - private static final String QUERY_OFFSET_METRIC = "query_hit_offset"; + private static final String QUERY_HIT_OFFSET_METRIC = "query_hit_offset"; private static final String QUERIES_METRIC = "queries"; - private static final String ACTIVE_QUERIES_METRIC = "active_queries"; private static final String PEAK_QPS_METRIC = "peak_qps"; private static final String DOCS_COVERED_METRIC = "documents_covered"; private static final String DOCS_TOTAL_METRIC = "documents_total"; - private static final String DEGRADED_METRIC = "degraded_queries"; + private static final String DEGRADED_QUERIES_METRIC = "degraded_queries"; private static final String RELEVANCE_AT_1_METRIC = "relevance.at_1"; private static final String RELEVANCE_AT_3_METRIC = "relevance.at_3"; private static final String RELEVANCE_AT_10_METRIC = "relevance.at_10"; - private final Counter queries; // basic counter - private final Counter failedQueries; // basic counter - private final Counter nullQueries; // basic counter - private final Counter illegalQueries; // basic counter - private final Value queryLatency; // mean pr 5 min + private final Counter queriesCounter; // basic counter + private final Counter failedQueriesCounter; // basic counter + private final Counter nullQueriesCounter; // basic counter + private final Counter illegalQueriesCounter; // basic counter + private final Value meanQueryLatency; // mean pr 5 min private final Value queryLatencyBuckets; private final Value maxQueryLatency; // separate to avoid name mangling @SuppressWarnings("unused") // all the work is done by the callback private final Value peakQPS; // peak 1s QPS - private final Counter emptyResults; // number of results containing no concrete hits + private final Counter emptyResultsCounter; // number of results containing no concrete hits private final Value hitsPerQuery; // mean number of hits per query + private final Value totalHitsPerQuery; private final PeakQpsReporter peakQpsReporter; // Naming of enums are reflected directly in metric dimensions and should not be changed as they are public API private enum DegradedReason { match_phase, adaptive_timeout, timeout, non_ideal_state } - private Metric metric; - private Map<String, Metric.Context> statePageOnlyContexts = new CopyOnWriteHashMap<>(); - private Map<String, Map<DegradedReason, Metric.Context>> degradedReasonContexts = new CopyOnWriteHashMap<>(); - private Map<String, Map<String, Metric.Context>> relevanceContexts = new CopyOnWriteHashMap<>(); - private java.util.Timer scheduler = new java.util.Timer(true); + private final Metric metric; + private final Map<String, Metric.Context> chainContexts = new CopyOnWriteHashMap<>(); + private final Map<String, Metric.Context> statePageOnlyContexts = new CopyOnWriteHashMap<>(); + private final Map<String, Map<DegradedReason, Metric.Context>> degradedReasonContexts = new CopyOnWriteHashMap<>(); + private final Map<String, Map<String, Metric.Context>> relevanceContexts = new CopyOnWriteHashMap<>(); + private final java.util.Timer scheduler = new java.util.Timer(true); private class PeakQpsReporter extends java.util.TimerTask { private long prevMaxQPSTime = System.currentTimeMillis(); @@ -127,17 +139,21 @@ public class StatisticsSearcher extends Searcher { this.peakQpsReporter = new PeakQpsReporter(); this.metric = metric; - queries = new Counter(QUERIES_METRIC, manager, false); - failedQueries = new Counter(FAILED_QUERIES_METRIC, manager, false); - nullQueries = new Counter("null_queries", manager, false); - illegalQueries = new Counter("illegal_queries", manager, false); - queryLatency = new Value(MEAN_QUERY_LATENCY_METRIC, manager, new Value.Parameters().setLogRaw(false).setLogMean(true).setNameExtension(false)); + queriesCounter = new Counter(QUERIES_METRIC, manager, false); + failedQueriesCounter = new Counter(FAILED_QUERIES_METRIC, manager, false); + nullQueriesCounter = new Counter("null_queries", manager, false); + illegalQueriesCounter = new Counter("illegal_queries", manager, false); + meanQueryLatency = new Value(MEAN_QUERY_LATENCY_METRIC, manager, new Value.Parameters().setLogRaw(false).setLogMean(true).setNameExtension(false)); maxQueryLatency = new Value(MAX_QUERY_LATENCY_METRIC, manager, new Value.Parameters().setLogRaw(false).setLogMax(true).setNameExtension(false)); queryLatencyBuckets = Value.buildValue(QUERY_LATENCY_METRIC, manager, null); peakQPS = new Value(PEAK_QPS_METRIC, manager, new Value.Parameters().setLogRaw(false).setLogMax(true).setNameExtension(false)); hitsPerQuery = new Value(HITS_PER_QUERY_METRIC, manager, new Value.Parameters().setLogRaw(false).setLogMean(true).setNameExtension(false)); - emptyResults = new Counter(EMPTY_RESULTS_METRIC, manager, false); + totalHitsPerQuery = new Value(TOTALHITS_PER_QUERY_METRIC, manager, new Value.Parameters().setLogRaw(false).setLogMean(true).setNameExtension(false)); + + emptyResultsCounter = new Counter(EMPTY_RESULTS_METRIC, manager, false); metricReceiver.declareGauge(QUERY_LATENCY_METRIC, Optional.empty(), new MetricSettings.Builder().histogram(true).build()); + metricReceiver.declareGauge(HITS_PER_QUERY_METRIC, Optional.empty(), new MetricSettings.Builder().histogram(true).build()); + metricReceiver.declareGauge(TOTALHITS_PER_QUERY_METRIC, Optional.empty(), new MetricSettings.Builder().histogram(true).build()); scheduler.schedule(peakQpsReporter, 1000, 1000); } @@ -152,11 +168,15 @@ public class StatisticsSearcher extends Searcher { peakQpsReporter.countQuery(); } - private Metric.Context getChainMetricContext(String chainName, String endpoint) { - Map<String, String> dimensions = new HashMap<>(); - dimensions.put("chain", chainName); - dimensions.put("endpoint", endpoint); - return this.metric.createContext(dimensions); + private Metric.Context getChainMetricContext(String chainName) { + Metric.Context context = chainContexts.get(chainName); + if (context == null) { + Map<String, String> dimensions = new HashMap<>(); + dimensions.put("chain", chainName); + context = this.metric.createContext(dimensions); + chainContexts.put(chainName, context); + } + return context; } private Metric.Context getDegradedMetricContext(String chainName, Coverage coverage) { @@ -223,11 +243,11 @@ public class StatisticsSearcher extends Searcher { return execution.search(query); } - Metric.Context metricContext = getChainMetricContext(execution.chain().getId().stringValue(), query.getHttpRequest().getHeader("Host")); + Metric.Context metricContext = getChainMetricContext(execution.chain().getId().stringValue()); incrQueryCount(metricContext); logQuery(query); - long start = System.currentTimeMillis(); // Start time, in millisecs. + long start_ns = getStartNanoTime(query); qps(metricContext); Result result; //handle exceptions thrown below in searchers @@ -238,14 +258,14 @@ public class StatisticsSearcher extends Searcher { throw e; } - long end = System.currentTimeMillis(); // Start time, in millisecs. - long latency = end - start; - if (latency >= 0) { - addLatency(latency, metricContext); + long end_ns = System.nanoTime(); // End time, in nanoseconds + long latency_ns = end_ns - start_ns; + if (latency_ns >= 0) { + addLatency(latency_ns, metricContext); } else { - getLogger().log(LogLevel.WARNING, - "Apparently negative latency measure, start: " + start - + ", end: " + end + ", for query: " + query.toString()); + getLogger().log(Level.WARNING, + "Apparently negative latency measure, start: " + start_ns + + ", end: " + end_ns + ", for query: " + query.toString()); } if (result.hits().getError() != null) { incrErrorCount(result, metricContext); @@ -255,18 +275,22 @@ public class StatisticsSearcher extends Searcher { if (queryCoverage != null) { if (queryCoverage.isDegraded()) { Metric.Context degradedContext = getDegradedMetricContext(execution.chain().getId().stringValue(), queryCoverage); - metric.add(DEGRADED_METRIC, 1, degradedContext); + metric.add(DEGRADED_QUERIES_METRIC, 1, degradedContext); } metric.add(DOCS_COVERED_METRIC, queryCoverage.getDocs(), metricContext); metric.add(DOCS_TOTAL_METRIC, queryCoverage.getActive(), metricContext); } int hitCount = result.getConcreteHitCount(); - hitsPerQuery.put((double) hitCount); + hitsPerQuery.put(hitCount); metric.set(HITS_PER_QUERY_METRIC, (double) hitCount, metricContext); - metric.set(TOTALHITS_PER_QUERY_METRIC, (double) result.getTotalHitCount(), metricContext); - metric.set(QUERY_OFFSET_METRIC, (double) (query.getHits() + query.getOffset()), metricContext); + + long totalHitCount = result.getTotalHitCount(); + totalHitsPerQuery.put(totalHitCount); + metric.set(TOTALHITS_PER_QUERY_METRIC, (double) totalHitCount, metricContext); + + metric.set(QUERY_HIT_OFFSET_METRIC, (double) (query.getHits() + query.getOffset()), metricContext); if (hitCount == 0) { - emptyResults.increment(); + emptyResultsCounter.increment(); metric.add(EMPTY_RESULTS_METRIC, 1, metricContext); } @@ -283,9 +307,10 @@ public class StatisticsSearcher extends Searcher { } } - private void addLatency(long latency, Metric.Context metricContext) { + private void addLatency(long latency_ns, Metric.Context metricContext) { + double latency = 0.000001 * latency_ns; //myStats.addLatency(latency); - queryLatency.put(latency); + meanQueryLatency.put(latency); metric.set(QUERY_LATENCY_METRIC, latency, metricContext); metric.set(MEAN_QUERY_LATENCY_METRIC, latency, metricContext); maxQueryLatency.put(latency); @@ -295,20 +320,20 @@ public class StatisticsSearcher extends Searcher { private void incrQueryCount(Metric.Context metricContext) { //myStats.incrQueryCnt(); - queries.increment(); + queriesCounter.increment(); metric.add(QUERIES_METRIC, 1, metricContext); } private void incrErrorCount(Result result, Metric.Context metricContext) { - failedQueries.increment(); + failedQueriesCounter.increment(); metric.add(FAILED_QUERIES_METRIC, 1, metricContext); if (result == null) // the chain threw an exception metric.add("error.unhandled_exception", 1, metricContext); else if (result.hits().getErrorHit().hasOnlyErrorCode(Error.NULL_QUERY.code)) - nullQueries.increment(); + nullQueriesCounter.increment(); else if (result.hits().getErrorHit().hasOnlyErrorCode(Error.ILLEGAL_QUERY.code)) - illegalQueries.increment(); + illegalQueriesCounter.increment(); } /** @@ -414,5 +439,15 @@ public class StatisticsSearcher extends Searcher { } } + /** + * Returns the relative start time from request was received by jdisc + */ + private static long getStartNanoTime(Query query) { + return Optional.ofNullable(query.getHttpRequest()) + .flatMap(httpRequest -> Optional.ofNullable(httpRequest.getJDiscRequest())) + .map(HttpRequest::relativeCreatedAtNanoTime) + .orElseGet(System::nanoTime); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 1e3f11f4f78..08ebd74da5a 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -7,10 +7,9 @@ import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.fs4.MapEncoder; -import com.yahoo.log.LogLevel; +import com.yahoo.language.process.Embedder; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.query.Highlight; -import com.yahoo.prelude.query.QueryException; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.dispatch.Dispatcher; @@ -57,6 +56,8 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -182,7 +183,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { //---------------- Tracing ---------------------------------------------------- - private static Logger log = Logger.getLogger(Query.class.getName()); + private static final Logger log = Logger.getLogger(Query.class.getName()); /** The time this query was created */ private long startTime; @@ -201,7 +202,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public static final CompoundName TIMEOUT = new CompoundName("timeout"); - private static QueryProfileType argumentType; + private static final QueryProfileType argumentType; static { argumentType = new QueryProfileType("native"); argumentType.setBuiltin(true); @@ -227,7 +228,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public static QueryProfileType getArgumentType() { return argumentType; } /** The aliases of query properties */ - private static Map<String, CompoundName> propertyAliases; + private static final Map<String, CompoundName> propertyAliases; static { Map<String,CompoundName> propertyAliasesBuilder = new HashMap<>(); addAliases(Query.getArgumentType(), propertyAliasesBuilder); @@ -317,7 +318,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * Creates a query from a request * * @param request the HTTP request from which this is created - * @param queryProfile the query profile to use for this query, or null if none. + * @param queryProfile the query profile to use for this query, or null if none */ public Query(HttpRequest request, CompiledQueryProfile queryProfile) { this(request, request.propertyMap(), queryProfile); @@ -326,27 +327,39 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** * Creates a query from a request * - * @param request the HTTP request from which this is created. - * @param requestMap the property map of the query. - * @param queryProfile the query profile to use for this query, or null if none. + * @param request the HTTP request from which this is created + * @param requestMap the property map of the query + * @param queryProfile the query profile to use for this query, or null if none */ public Query(HttpRequest request, Map<String, String> requestMap, CompiledQueryProfile queryProfile) { super(new QueryPropertyAliases(propertyAliases)); this.httpRequest = request; - init(requestMap, queryProfile); + init(requestMap, queryProfile, Embedder.throwsOnUse); + } + + // TODO: Deprecate most constructors above here + + private Query(Builder builder) { + this(builder.getRequest(), builder.getRequestMap(), builder.getQueryProfile(), builder.getEmbedder()); + } + + private Query(HttpRequest request, Map<String, String> requestMap, CompiledQueryProfile queryProfile, Embedder embedder) { + super(new QueryPropertyAliases(propertyAliases)); + this.httpRequest = request; + init(requestMap, queryProfile, embedder); } - private void init(Map<String, String> requestMap, CompiledQueryProfile queryProfile) { - startTime = System.currentTimeMillis(); + private void init(Map<String, String> requestMap, CompiledQueryProfile queryProfile, Embedder embedder) { + startTime = httpRequest.getJDiscRequest().creationTime(TimeUnit.MILLISECONDS); if (queryProfile != null) { // Move all request parameters to the query profile just to validate that the parameter settings are legal - Properties queryProfileProperties = new QueryProfileProperties(queryProfile); + Properties queryProfileProperties = new QueryProfileProperties(queryProfile, embedder); properties().chain(queryProfileProperties); // TODO: Just checking legality rather than actually setting would be faster setPropertiesFromRequestMap(requestMap, properties(), true); // Adds errors to the query for illegal set attempts // Create the full chain - properties().chain(new QueryProperties(this, queryProfile.getRegistry())). + properties().chain(new QueryProperties(this, queryProfile.getRegistry(), embedder)). chain(new ModelObjectMap()). chain(new RequestContextProperties(requestMap)). chain(queryProfileProperties). @@ -365,7 +378,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } else { // bypass these complications if there is no query profile to get values from and validate against properties(). - chain(new QueryProperties(this, CompiledQueryProfileRegistry.empty)). + chain(new QueryProperties(this, CompiledQueryProfileRegistry.empty, embedder)). chain(new PropertyMap()). chain(new DefaultProperties()); setPropertiesFromRequestMap(requestMap, properties(), false); @@ -413,11 +426,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { if (field.getType() == FieldType.genericQueryProfileType) { // Generic map CompoundName fullName = prefix.append(field.getName()); for (Map.Entry<String, Object> entry : originalProperties.listProperties(fullName, context).entrySet()) { - try { - properties().set(fullName.append(entry.getKey()), entry.getValue(), context); - } catch (IllegalArgumentException e) { - throw new QueryException("Invalid request parameter", e); - } + properties().set(fullName.append(entry.getKey()), entry.getValue(), context); } } else if (field.getType() instanceof QueryProfileFieldType) { // Nested arguments @@ -427,11 +436,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { CompoundName fullName = prefix.append(field.getName()); Object value = originalProperties.get(fullName, context); if (value != null) { - try { - properties().set(fullName, value, context); - } catch (IllegalArgumentException e) { - throw new QueryException("Invalid request parameter", e); - } + properties().set(fullName, value, context); } } } @@ -440,13 +445,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Calls properties.set on all entries in requestMap */ private void setPropertiesFromRequestMap(Map<String, String> requestMap, Properties properties, boolean ignoreSelect) { for (var entry : requestMap.entrySet()) { - try { - if (ignoreSelect && entry.getKey().equals(Select.SELECT)) continue; - properties.set(entry.getKey(), entry.getValue(), requestMap); - } - catch (IllegalArgumentException e) { - throw new QueryException("Invalid request parameter", e); - } + if (ignoreSelect && entry.getKey().equals(Select.SELECT)) continue; + properties.set(entry.getKey(), entry.getValue(), requestMap); } } @@ -663,7 +663,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { // getQueryTree isn't exception safe try { queryTree = model.getQueryTree().toString(); - } catch (Exception e) { + } catch (Exception | StackOverflowError e) { queryTree = "[Could not parse user input: " + model.getQueryString() + "]"; } return "query '" + queryTree + "'"; @@ -675,7 +675,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { // getQueryTree isn't exception safe try { queryTree = model.getQueryTree().toString(); - } catch (Exception e) { + } catch (Exception | StackOverflowError e) { queryTree = "Could not parse user input: " + model.getQueryString(); } return "query=[" + queryTree + "]" + " offset=" + getOffset() + " hits=" + getHits() + "]"; @@ -726,7 +726,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { if (includeQuery) message += ": [" + queryTreeText() + "]"; - log.log(LogLevel.DEBUG,message); + log.log(Level.FINE,message); // Pass 0 as traceLevel as the trace level check is already done above, // and it is not propagated to trace until execution has started @@ -995,6 +995,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { clone.properties().setParentQuery(clone); assert (clone.properties().getParentQuery() == clone); + clone.setTimeout(getTimeout()); clone.setTraceLevel(getTraceLevel()); clone.setExplainLevel(getExplainLevel()); clone.setHits(getHits()); @@ -1124,4 +1125,59 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { getRanking().prepare(); } + public static class Builder { + + private HttpRequest request = null; + private Map<String, String> requestMap = null; + private CompiledQueryProfile queryProfile = null; + private Embedder embedder = Embedder.throwsOnUse; + + public Builder setRequest(String query) { + request = HttpRequest.createTestRequest(query, com.yahoo.jdisc.http.HttpRequest.Method.GET); + return this; + } + + public Builder setRequest(HttpRequest request) { + this.request = request; + return this; + } + + public HttpRequest getRequest() { + if (request == null) + return HttpRequest.createTestRequest("", com.yahoo.jdisc.http.HttpRequest.Method.GET); + return request; + } + + /** Sets the request mao to use explicitly. If not set, the request map will be getRequest().propertyMap() */ + public Builder setRequestMap(Map<String, String> requestMap) { + this.requestMap = requestMap; + return this; + } + + public Map<String, String> getRequestMap() { + if (requestMap == null) + return getRequest().propertyMap(); + return requestMap; + } + + public Builder setQueryProfile(CompiledQueryProfile queryProfile) { + this.queryProfile = queryProfile; + return this; + } + + /** Returns the query profile of this query, or null if none. */ + public CompiledQueryProfile getQueryProfile() { return queryProfile; } + + public Builder setEmbedder(Embedder embedder) { + this.embedder = embedder; + return this; + } + + public Embedder getEmbedder() { return embedder; } + + /** Creates a new query from this builder. No properties are required to before calling this. */ + public Query build() { return new Query(this); } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/Result.java b/container-search/src/main/java/com/yahoo/search/Result.java index 4080b09f40b..2e9018217bb 100644 --- a/container-search/src/main/java/com/yahoo/search/Result.java +++ b/container-search/src/main/java/com/yahoo/search/Result.java @@ -21,6 +21,8 @@ import java.util.Iterator; * result items, as well as further HitGroups, making up a <i>composite</i> structure. This allows the hits of a result * to be hierarchically organized. A Hit is polymorphic and may contain any kind of information deemed * an approriate partial answer to the Query. + * <p> + * Do not cache this as it holds references to objects that should be garbage collected. * * @author bratseth */ @@ -51,7 +53,7 @@ public final class Result extends com.yahoo.processing.Response implements Clone * Headers containing "envelope" meta information to be returned with this result. * Used for HTTP getHeaders when the return protocol is HTTP. */ - private ListMap<String,String> headers = null; + private ListMap<String, String> headers = null; /** Creates a new Result where the top level hit group has id "toplevel" */ public Result(Query query) { @@ -66,7 +68,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone * @param query the query which produced this result * @param hits the hit container which this will return from {@link #hits()} */ - @SuppressWarnings("deprecation") public Result(Query query, HitGroup hits) { super(query); if (query==null) throw new NullPointerException("The query reference in a result cannot be null"); @@ -89,7 +90,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone * with a result. It should <b>always</b> be called when adding * hits from a result, but there is no constraints on the order of the calls. */ - @SuppressWarnings("deprecation") public void mergeWith(Result result) { totalHitCount += result.getTotalHitCount(); deepHitCount += result.getDeepHitCount(); diff --git a/container-search/src/main/java/com/yahoo/search/Searcher.java b/container-search/src/main/java/com/yahoo/search/Searcher.java index 5fefe9d2468..cd6b7167f08 100644 --- a/container-search/src/main/java/com/yahoo/search/Searcher.java +++ b/container-search/src/main/java/com/yahoo/search/Searcher.java @@ -73,6 +73,7 @@ public abstract class Searcher extends Processor { // Note to developers: If you think you should add something here you are probably wrong // Create a subclass containing the new method instead. + private final Logger logger = Logger.getLogger(getClass().getName()); public Searcher() {} diff --git a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java index 0d491d2f0c1..8eee7c11d3e 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java @@ -44,7 +44,7 @@ public abstract class BaseNodeMonitor<T> { protected MonitorConfiguration configuration; /** Is the node we monitor part of an internal Vespa cluster or not */ - private boolean internal; + private final boolean internal; public BaseNodeMonitor(boolean internal) { this.internal=internal; diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 15cf4995b77..c9b8aeee417 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -25,9 +25,9 @@ import java.util.logging.Logger; */ public class ClusterMonitor<T> { - private final MonitorConfiguration configuration = new MonitorConfiguration(); + private static final Logger log = Logger.getLogger(ClusterMonitor.class.getName()); - private static Logger log = Logger.getLogger(ClusterMonitor.class.getName()); + private final MonitorConfiguration configuration = new MonitorConfiguration(); private final NodeManager<T> nodeManager; @@ -62,6 +62,8 @@ public class ClusterMonitor<T> { /** Returns the configuration of this cluster monitor */ public MonitorConfiguration getConfiguration() { return configuration; } + public boolean isClosed() { return closed.get(); } + /** * Adds a new node for monitoring. * The object representing the node must diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 2d05168731a..db79c610dc7 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -3,7 +3,7 @@ package com.yahoo.search.cluster; import com.yahoo.component.ComponentId; import com.yahoo.container.protect.Error; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.yolean.Exceptions; @@ -71,7 +71,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod /** Pinging a node, called from ClusterMonitor */ @Override public final void ping(ClusterMonitor<T> clusterMonitor, T p, Executor executor) { - log(LogLevel.FINE, "Sending ping to: ", p); + log(Level.FINE, "Sending ping to: ", p); Pinger pinger = new Pinger(p); FutureTask<Pong> future = new FutureTask<>(pinger); @@ -97,10 +97,10 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod if (pong.badResponse()) { clusterMonitor.failed(p, pong.error().get()); - log(LogLevel.FINE, "Failed ping - ", pong); + log(Level.FINE, "Failed ping - ", pong); } else { clusterMonitor.responded(p); - log(LogLevel.FINE, "Answered ping - ", p); + log(Level.FINE, "Answered ping - ", p); } if (logThrowable != null) { @@ -173,7 +173,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod if (result.hits().getError().getCode() == Error.TIMEOUT.code) return result; // Retry is unlikely to help - log(LogLevel.FINER, "No result, checking for timeout."); + log(Level.FINER, "No result, checking for timeout."); tries++; connection = nodes.select(code, tries); } while (tries < nodes.getNodeCount()); @@ -211,7 +211,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod try { result = search(query, execution, connection); } catch (RuntimeException e) { //TODO: Exceptions should not be used to signal backend communication errors - log(LogLevel.WARNING, "An exception occurred while invoking backend searcher.", e); + log(Level.WARNING, "An exception occurred while invoking backend searcher.", e); result = new Result(query, ErrorMessage.createBackendCommunicationError("Failed calling " + connection + " in " + this + " for " + query + ": " + Exceptions.toMessageString(e))); @@ -220,14 +220,6 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod if (result == null) result = new Result(query, ErrorMessage.createBackendCommunicationError("No result returned in " + this + " from " + connection + " for " + query)); - - if (result.hits().getError() != null) { - log(LogLevel.FINE, "FAILED: ", query); - } else if ( ! result.isCached()) { - log(LogLevel.FINE, "WORKING: ", query); - } else { - log(LogLevel.FINE, "CACHE HIT: ", query); - } return result; } @@ -263,13 +255,6 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod result.hits().addError(ErrorMessage.createBackendCommunicationError("Error filling " + result + " from " + connection + ": " + Exceptions.toMessageString(e))); } - if (result.hits().getError() != null) { - log(LogLevel.FINE, "FAILED: ", result.getQuery()); - } else if ( ! result.isCached()) { - log(LogLevel.FINE, "WORKING: ", result.getQuery()); - } else { - log(LogLevel.FINE, "CACHE HIT: " + result.getQuery()); - } } /** diff --git a/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java b/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java index 46752b0bedb..6c83e1c64e3 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java @@ -11,6 +11,7 @@ package com.yahoo.search.cluster; public class Hasher<T> { public static class NodeFactor<T> { + private final T node; /** diff --git a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java index 21e5fe3bc7f..95f51b374d6 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java @@ -8,62 +8,44 @@ package com.yahoo.search.cluster; */ public class MonitorConfiguration { - /** - * The interval in ms between consecutive checks of the monitored - * nodes - */ - private long checkInterval=1000; + /** The interval in ms between consecutive checks of the monitored nodes */ + private long checkInterval = 1000; - /** - * The number of milliseconds to attempt to complete a request - * before giving up - */ + /** The number of milliseconds to attempt to complete a request before giving up */ private final long requestTimeout = 980; - /** - * The number of milliseconds a node is allowed to fail before we - * mark it as not working - */ - private long failLimit=5000; + /** The number of milliseconds a node is allowed to fail before we mark it as not working */ + private long failLimit = 5000; - /** - * Sets the interval between each ping of idle or failing nodes - * Default is 1000ms - */ - public void setCheckInterval(long intervalMs) { - this.checkInterval=intervalMs; - } + /** Sets the interval between each ping of idle or failing nodes. Default is 1000 ms. */ + @Deprecated // TODO: Remove on Vespa 8 + public void setCheckInterval(long intervalMs) { this.checkInterval = intervalMs; } - /** - * Returns the interval between each ping of idle or failing nodes - * Default is 1000ms - */ - public long getCheckInterval() { - return checkInterval; - } + /** Returns the interval between each ping of idle or failing nodes. Default is 1000 ms. */ + public long getCheckInterval() { return checkInterval; } /** - * Sets the number of times a failed node must respond before it is put - * back in service. Default is 3. - * @deprecated Will go away in Vespa 8 + * Sets the number of times a failed node must respond before it is put back in service. Default is 3. + * + * @deprecated will go away in Vespa 8 */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 8 public void setResponseAfterFailLimit(int responseAfterFailLimit) { } /** - * Sets the number of ms a node (failing or working) is allowed to - * stay idle before it is pinged. Default is 3000 + * Sets the number of ms a node (failing or working) is allowed to stay idle before it is pinged. Default is 3000. + * * @deprecated Will go away in Vespa 8 */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 8 public void setIdleLimit(int idleLimit) { } /** - * Gets the number of ms a node (failing or working) - * is allowed to stay idle before it is pinged. Default is 3000 + * Gets the number of ms a node (failing or working) is allowed to stay idle before it is pinged. Default is 3000. + * * @deprecated Will go away in Vespa 8 */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 8 public long getIdleLimit() { return 3000; } @@ -78,6 +60,7 @@ public class MonitorConfiguration { * Sets the number of milliseconds a node is allowed to fail before we * mark it as not working */ + @Deprecated // TODO: Remove on Vespa 8 public void setFailLimit(long failLimit) { this.failLimit=failLimit; } /** @@ -91,25 +74,27 @@ public class MonitorConfiguration { * in quarantine. Once in quarantine it won't be put back in * productuion before quarantineTime has expired even if it is * working. Default is 3 + * * @deprecated Will go away in Vespa 8 */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 8 public void setFailQuarantineLimit(int failQuarantineLimit) { } /** - * The number of ms an unstable node is quarantined. Default is - * 100*60*60 + * The number of ms an unstable node is quarantined. Default is 100*60*60 + * * @deprecated Will go away in Vespa 8 */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 8 public void setQuarantineTime(long quarantineTime) { } + @Override public String toString() { return "monitor configuration [" + - "checkInterval: " + checkInterval + - " requestTimeout " + requestTimeout + - " failLimit " + failLimit + - "]"; + "checkInterval: " + checkInterval + + " requestTimeout " + requestTimeout + + " failLimit " + failLimit + + "]"; } } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java index 481f1e1b5a5..836c71089c1 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java @@ -7,7 +7,7 @@ import java.util.concurrent.Executor; * Must be implemented by a node collection which wants * it's node state monitored by a ClusterMonitor * - * @author bratseth + * @author bratseth */ public interface NodeManager<T> { @@ -20,9 +20,10 @@ public interface NodeManager<T> { /** * Called when a node should be pinged. * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded + * * @deprecated Use ping(ClusterMonitor clusterMonitor, T node, Executor executor) instead. */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 8 default void ping(T node, Executor executor) { throw new IllegalStateException("If you have not overrriden ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor), you should at least have overriden this method."); } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java index ccf3e863ff3..d17f6bfbaa8 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java @@ -52,8 +52,8 @@ public class TrafficNodeMonitor<T> extends BaseNodeMonitor<T> { * Called when a response is received from this node. */ public void responded() { - respondedAt=now(); - succeededAt=respondedAt; + respondedAt = now(); + succeededAt = respondedAt; setWorking(true,"Responds correctly"); } @@ -69,20 +69,20 @@ public class TrafficNodeMonitor<T> extends BaseNodeMonitor<T> { atStartUp = false; if (this.isWorking == working) return; // Old news - if (explanation==null) { - explanation=""; + if (explanation == null) { + explanation = ""; } else { - explanation=": " + explanation; + explanation = ": " + explanation; } if (working) { log.info("Putting " + node + " in service" + explanation); } else { log.warning("Taking " + node + " out of service" + explanation); - failedAt=now(); + failedAt = now(); } - this.isWorking=working; + this.isWorking = working; } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java index 515d6249fd8..9329f4a6819 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java @@ -12,6 +12,7 @@ import java.util.function.BiConsumer; * @author ollivir */ public abstract class CloseableInvoker implements Closeable { + protected abstract void release(); private BiConsumer<Boolean, Long> teardown = null; @@ -35,4 +36,5 @@ public abstract class CloseableInvoker implements Closeable { } release(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index abe6fffba39..dd3c760d2f4 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -4,6 +4,7 @@ package com.yahoo.search.dispatch; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.ComponentId; +import com.yahoo.compress.Compressor; import com.yahoo.container.handler.VipStatus; import com.yahoo.jdisc.Metric; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; +import java.util.stream.Collectors; /** * A dispatcher communicates with search nodes to perform queries and fill hits. @@ -48,6 +50,7 @@ public class Dispatcher extends AbstractComponent { public static final String DISPATCH = "dispatch"; private static final String INTERNAL = "internal"; private static final String PROTOBUF = "protobuf"; + private static final String TOP_K_PROBABILITY = "topKProbability"; private static final String INTERNAL_METRIC = "dispatch_internal"; @@ -56,9 +59,12 @@ public class Dispatcher extends AbstractComponent { /** If enabled, search queries will use protobuf rpc */ public static final CompoundName dispatchProtobuf = CompoundName.fromComponents(DISPATCH, PROTOBUF); + /** If set will control computation of how many hits will be fetched from each partition.*/ + public static final CompoundName topKProbability = CompoundName.fromComponents(DISPATCH, TOP_K_PROBABILITY); + /** A model of the search cluster this dispatches to */ private final SearchCluster searchCluster; - private final ClusterMonitor clusterMonitor; + private final ClusterMonitor<Node> clusterMonitor; private final LoadBalancer loadBalancer; @@ -77,6 +83,7 @@ public class Dispatcher extends AbstractComponent { argumentType.setBuiltin(true); argumentType.addField(new FieldDescription(INTERNAL, FieldType.booleanType)); argumentType.addField(new FieldDescription(PROTOBUF, FieldType.booleanType)); + argumentType.addField(new FieldDescription(TOP_K_PROBABILITY, FieldType.doubleType)); argumentType.freeze(); } @@ -99,7 +106,7 @@ public class Dispatcher extends AbstractComponent { } /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */ - protected Dispatcher(ClusterMonitor clusterMonitor, + protected Dispatcher(ClusterMonitor<Node> clusterMonitor, SearchCluster searchCluster, DispatchConfig dispatchConfig, InvokerFactory invokerFactory, @@ -116,21 +123,30 @@ public class Dispatcher extends AbstractComponent { this.metricContext = metric.createContext(null); this.maxHitsPerNode = dispatchConfig.maxHitsPerNode(); searchCluster.addMonitoring(clusterMonitor); + Thread warmup = new Thread(() -> warmup(dispatchConfig.warmuptime())); + warmup.start(); try { while ( ! searchCluster.hasInformationAboutAllNodes()) { Thread.sleep(1); } + warmup.join(); } catch (InterruptedException e) {} - /* - * No we have information from all nodes and a ping iteration has completed. - * Instead of waiting until next ping interval to update coverage and group state, - * we should compute the state ourselves, so that when the dispatcher is ready the state - * of its groups are also known. - */ + // Now we have information from all nodes and a ping iteration has completed. + // Instead of waiting until next ping interval to update coverage and group state, + // we should compute the state ourselves, so that when the dispatcher is ready the state + // of its groups are also known. searchCluster.pingIterationCompleted(); } + /** + * Will run important code in order to trigger JIT compilation and avoid cold start issues. + * Currently warms up lz4 compression code. + */ + private static long warmup(double seconds) { + return new Compressor().warmup(seconds); + } + /** Returns the search cluster this dispatches to */ public SearchCluster searchCluster() { return searchCluster; @@ -138,7 +154,7 @@ public class Dispatcher extends AbstractComponent { @Override public void deconstruct() { - /* The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. */ + // The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. clusterMonitor.shutdown(); invokerFactory.release(); } @@ -168,8 +184,8 @@ public class Dispatcher extends AbstractComponent { if (nodes.isEmpty()) return Optional.empty(); query.trace(false, 2, "Dispatching with search path ", searchPath); - return invokerFactory.createSearchInvoker(searcher, query, - OptionalInt.empty(), + return invokerFactory.createSearchInvoker(searcher, + query, nodes, true, maxHitsPerNode); @@ -185,8 +201,7 @@ public class Dispatcher extends AbstractComponent { query.trace(false, 2, "Dispatching to ", node); return invokerFactory.createSearchInvoker(searcher, query, - OptionalInt.empty(), - Arrays.asList(node), + List.of(node), true, maxHitsPerNode) .orElseThrow(() -> new IllegalStateException("Could not dispatch directly to " + node)); @@ -195,7 +210,7 @@ public class Dispatcher extends AbstractComponent { int covered = searchCluster.groupsWithSufficientCoverage(); int groups = searchCluster.orderedGroups().size(); int max = Integer.min(Integer.min(covered + 1, groups), MAX_GROUP_SELECTION_ATTEMPTS); - Set<Integer> rejected = null; + Set<Integer> rejected = rejectGroupBlockingFeed(searchCluster.orderedGroups()); for (int i = 0; i < max; i++) { Optional<Group> groupInCluster = loadBalancer.takeGroup(rejected); if (groupInCluster.isEmpty()) break; // No groups available @@ -204,7 +219,6 @@ public class Dispatcher extends AbstractComponent { boolean acceptIncompleteCoverage = (i == max - 1); Optional<SearchInvoker> invoker = invokerFactory.createSearchInvoker(searcher, query, - OptionalInt.of(group.id()), group.nodes(), acceptIncompleteCoverage, maxHitsPerNode); @@ -224,4 +238,21 @@ public class Dispatcher extends AbstractComponent { throw new IllegalStateException("No suitable groups to dispatch query. Rejected: " + rejected); } + /** + * We want to avoid groups blocking feed because their data may be out of date. + * If there is a single group blocking feed, we want to reject it. + * If multiple groups are blocking feed we should use them anyway as we may not have remaining + * capacity otherwise. Same if there are no other groups. + * + * @return a modifiable set containing the single group to reject, or null otherwise + */ + private Set<Integer> rejectGroupBlockingFeed(List<Group> groups) { + if (groups.size() == 1) return null; + List<Group> groupsRejectingFeed = groups.stream().filter(Group::isBlockingWrites).collect(Collectors.toList()); + if (groupsRejectingFeed.size() != 1) return null; + Set<Integer> rejected = new HashSet<>(); + rejected.add(groupsRejectingFeed.get(0).id()); + return rejected; + } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java index dd4c4494ac5..8b7714aaf3b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java @@ -10,7 +10,8 @@ import com.yahoo.search.Result; * @author ollivir */ public abstract class FillInvoker extends CloseableInvoker { - /** Retrieve document summaries for the unfilled hits in the given {@link Result} */ + + /** Retrieves document summaries for the unfilled hits in the given {@link Result} */ public void fill(Result result, String summaryClass) { sendFillRequest(result, summaryClass); getFillResults(result, summaryClass); @@ -19,4 +20,5 @@ public abstract class FillInvoker extends CloseableInvoker { protected abstract void getFillResults(Result result, String summaryClass); protected abstract void sendFillRequest(Result result, String summaryClass); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index cec3e94d551..fb04c8299e9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -3,6 +3,7 @@ package com.yahoo.search.dispatch; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.result.Coverage; import com.yahoo.search.result.ErrorMessage; @@ -41,6 +42,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM private final Set<SearchInvoker> invokers; private final SearchCluster searchCluster; + private final Group group; private final LinkedBlockingQueue<SearchInvoker> availableForProcessing; private final Set<Integer> alreadyFailedNodes; private Query query; @@ -59,11 +61,15 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM private boolean timedOut = false; private boolean degradedByMatchPhase = false; - public InterleavedSearchInvoker(Collection<SearchInvoker> invokers, SearchCluster searchCluster, Set<Integer> alreadyFailedNodes) { + public InterleavedSearchInvoker(Collection<SearchInvoker> invokers, + SearchCluster searchCluster, + Group group, + Set<Integer> alreadyFailedNodes) { super(Optional.empty()); this.invokers = Collections.newSetFromMap(new IdentityHashMap<>()); this.invokers.addAll(invokers); this.searchCluster = searchCluster; + this.group = group; this.availableForProcessing = newQueue(); this.alreadyFailedNodes = alreadyFailedNodes; } @@ -74,23 +80,33 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM * will be adjusted accordingly. */ @Override - protected void sendSearchRequest(Query query) throws IOException { + protected Object sendSearchRequest(Query query, Object unusedContext) throws IOException { this.query = query; invokers.forEach(invoker -> invoker.setMonitor(this)); deadline = currentTime() + query.getTimeLeft(); int originalHits = query.getHits(); int originalOffset = query.getOffset(); - query.setHits(query.getHits() + query.getOffset()); + int neededHits = originalHits + originalOffset; + int q = neededHits; + if (group.isBalanced() && !group.isSparse()) { + Double topkProbabilityOverrride = query.properties().getDouble(Dispatcher.topKProbability); + q = (topkProbabilityOverrride != null) + ? searchCluster.estimateHitsToFetch(neededHits, invokers.size(), topkProbabilityOverrride) + : searchCluster.estimateHitsToFetch(neededHits, invokers.size()); + } + query.setHits(q); query.setOffset(0); + Object context = null; for (SearchInvoker invoker : invokers) { - invoker.sendSearchRequest(query); + context = invoker.sendSearchRequest(query, context); askedNodes++; } query.setHits(originalHits); query.setOffset(originalOffset); + return null; } @Override @@ -98,6 +114,8 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM InvokerResult result = new InvokerResult(query, query.getHits()); List<LeanHit> merged = Collections.emptyList(); long nextTimeout = query.getTimeLeft(); + boolean extraDebug = (query.getOffset() == 0) && (query.getHits() == 7) && log.isLoggable(java.util.logging.Level.FINE); + List<InvokerResult> processed = new ArrayList<>(); try { while (!invokers.isEmpty() && nextTimeout >= 0) { SearchInvoker invoker = availableForProcessing.poll(nextTimeout, TimeUnit.MILLISECONDS); @@ -105,7 +123,11 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM log.fine(() -> "Search timed out with " + askedNodes + " requests made, " + answeredNodes + " responses received"); break; } else { - merged = mergeResult(result.getResult(), invoker.getSearchResult(execution), merged); + InvokerResult toMerge = invoker.getSearchResult(execution); + if (extraDebug) { + processed.add(toMerge); + } + merged = mergeResult(result.getResult(), toMerge, merged); ejectInvoker(invoker); } nextTimeout = nextTimeout(); @@ -116,6 +138,33 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM insertNetworkErrors(result.getResult()); result.getResult().setCoverage(createCoverage()); + + if (extraDebug && merged.size() > 0) { + int firstPartId = merged.get(0).getPartId(); + for (int index = 1; index < merged.size(); index++) { + if (merged.get(index).getPartId() != firstPartId) { + extraDebug = false; + log.fine("merged["+index+"/"+merged.size()+"] from partId "+merged.get(index).getPartId()+", first "+firstPartId); + break; + } + } + } + if (extraDebug) { + log.fine("Interleaved "+processed.size()+" results"); + for (int pIdx = 0; pIdx < processed.size(); ++pIdx) { + var p = processed.get(pIdx); + log.fine("InvokerResult "+pIdx+" total hits "+p.getResult().getTotalHitCount()); + var lean = p.getLeanHits(); + for (int idx = 0; idx < lean.size(); ++idx) { + var hit = lean.get(idx); + log.fine("lean hit "+idx+" relevance "+hit.getRelevance()+" partid "+hit.getPartId()); + } + } + for (int mIdx = 0; mIdx < merged.size(); ++mIdx) { + var hit = merged.get(mIdx); + log.fine("merged hit "+mIdx+" relevance "+hit.getRelevance()+" partid "+hit.getPartId()); + } + } int needed = query.getOffset() + query.getHits(); for (int index = query.getOffset(); (index < merged.size()) && (index < needed); index++) { result.getLeanHits().add(merged.get(index)); @@ -213,15 +262,15 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM int indexCurrent = 0; int indexPartial = 0; while (indexCurrent < current.size() && indexPartial < partial.size() && merged.size() < needed) { - LeanHit incommingHit = partial.get(indexPartial); + LeanHit incomingHit = partial.get(indexPartial); LeanHit currentHit = current.get(indexCurrent); - int cmpRes = currentHit.compareTo(incommingHit); + int cmpRes = currentHit.compareTo(incomingHit); if (cmpRes < 0) { merged.add(currentHit); indexCurrent++; } else if (cmpRes > 0) { - merged.add(incommingHit); + merged.add(incomingHit); indexPartial++; } else { // Duplicates merged.add(currentHit); @@ -230,10 +279,12 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } } while ((indexCurrent < current.size()) && (merged.size() < needed)) { - merged.add(current.get(indexCurrent++)); + LeanHit currentHit = current.get(indexCurrent++); + merged.add(currentHit); } while ((indexPartial < partial.size()) && (merged.size() < needed)) { - merged.add(partial.get(indexPartial++)); + LeanHit incomingHit = partial.get(indexPartial++); + merged.add(incomingHit); } return merged; } @@ -321,4 +372,8 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM protected LinkedBlockingQueue<SearchInvoker> newQueue() { return new LinkedBlockingQueue<>(); } + + // For testing + Collection<SearchInvoker> invokers() { return invokers; } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index 1c3a90ac6ab..e602afadcfb 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -4,6 +4,7 @@ package com.yahoo.search.dispatch; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.result.Coverage; @@ -13,7 +14,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.OptionalInt; import java.util.Set; /** @@ -35,23 +35,22 @@ public abstract class InvokerFactory { public abstract FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result); /** - * Create a {@link SearchInvoker} for a list of content nodes. + * Creates a {@link SearchInvoker} for a list of content nodes. * * @param searcher the searcher processing the query * @param query the search query being processed - * @param groupId the id of the node group to which the nodes belong - * @param nodes pre-selected list of content nodes + * @param nodes pre-selected list of content nodes, all in a group or a subset of a group * @param acceptIncompleteCoverage if some of the nodes are unavailable and this parameter is * false, verify that the remaining set of nodes has sufficient coverage - * @return Optional containing the SearchInvoker or empty if some node in the + * @return the invoker or empty if some node in the * list is invalid and the remaining coverage is not sufficient */ - public Optional<SearchInvoker> createSearchInvoker(VespaBackEndSearcher searcher, - Query query, - OptionalInt groupId, - List<Node> nodes, - boolean acceptIncompleteCoverage, - int maxHits) { + Optional<SearchInvoker> createSearchInvoker(VespaBackEndSearcher searcher, + Query query, + List<Node> nodes, + boolean acceptIncompleteCoverage, + int maxHits) { + Group group = searchCluster.group(nodes.get(0).group()).get(); // Nodes must be of the same group List<SearchInvoker> invokers = new ArrayList<>(nodes.size()); Set<Integer> failed = null; for (Node node : nodes) { @@ -79,10 +78,10 @@ public abstract class InvokerFactory { success.add(node); } } - if ( ! searchCluster.isPartialGroupCoverageSufficient(groupId, success) && !acceptIncompleteCoverage) { + if ( ! searchCluster.isPartialGroupCoverageSufficient(success) && !acceptIncompleteCoverage) { return Optional.empty(); } - if(invokers.size() == 0) { + if (invokers.isEmpty()) { return Optional.of(createCoverageErrorInvoker(nodes, failed)); } } @@ -90,7 +89,7 @@ public abstract class InvokerFactory { if (invokers.size() == 1 && failed == null) { return Optional.of(invokers.get(0)); } else { - return Optional.of(new InterleavedSearchInvoker(invokers, searchCluster, failed)); + return Optional.of(new InterleavedSearchInvoker(invokers, searchCluster, group, failed)); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java index 94c347a6927..2723429c0cf 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java @@ -12,14 +12,19 @@ import java.util.List; /** * Wraps a Result and a flat, skinny hit list + * + * @author baldersheim */ public class InvokerResult { + private final Result result; private final List<LeanHit> leanHits; + public InvokerResult(Result result) { this.result = result; this.leanHits = Collections.emptyList(); } + public InvokerResult(Query query, int expectedHits) { result = new Result(query); leanHits = new ArrayList<>(expectedHits); @@ -32,6 +37,7 @@ public class InvokerResult { public List<LeanHit> getLeanHits() { return leanHits; } + void complete() { Query query = result.getQuery(); Sorting sorting = query.getRanking().getSorting(); @@ -47,4 +53,5 @@ public class InvokerResult { } leanHits.clear(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java index 86f1999d8b4..df8fb2f29fa 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java @@ -4,7 +4,11 @@ package com.yahoo.search.dispatch; import java.util.Arrays; +/** + * @author baldersheim + */ public class LeanHit implements Comparable<LeanHit> { + private final byte [] gid; private final double relevance; private final byte [] sortData; @@ -12,19 +16,16 @@ public class LeanHit implements Comparable<LeanHit> { private final int distributionKey; public LeanHit(byte [] gid, int partId, int distributionKey, double relevance) { - this.gid = gid; - this.relevance = Double.isNaN(relevance) ? Double.NEGATIVE_INFINITY : relevance; - this.sortData = null; - this.partId = partId; - this.distributionKey = distributionKey; + this(gid, partId, distributionKey, relevance, null); } - public LeanHit(byte [] gid, int partId, int distributionKey, byte [] sortData) { + public LeanHit(byte [] gid, int partId, int distributionKey, double relevance, byte [] sortData) { this.gid = gid; - this.relevance = 0.0; + this.relevance = Double.isNaN(relevance) ? Double.NEGATIVE_INFINITY : relevance; this.sortData = sortData; this.partId = partId; this.distributionKey = distributionKey; } + public double getRelevance() { return relevance; } public byte [] getGid() { return gid; } public byte [] getSortData() { return sortData; } @@ -53,4 +54,5 @@ public class LeanHit implements Comparable<LeanHit> { int vr = (int) right[i] & 0xFF; return vl - vr; } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java index 210ab5777d2..ebde2ffc611 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java @@ -43,8 +43,8 @@ public class LoadBalancer { } /** - * Select and allocate the search cluster group which is to be used for the next search query. Callers <b>must</b> call - * {@link #releaseGroup} symmetrically for each taken allocation. + * Select and allocate the search cluster group which is to be used for the next search query. + * Callers <b>must</b> call {@link #releaseGroup} symmetrically for each taken allocation. * * @param rejectedGroups if not null, the load balancer will only return groups with IDs not in the set * @return the node group to target, or <i>empty</i> if the internal dispatch logic cannot be used @@ -76,7 +76,7 @@ public class LoadBalancer { synchronized (this) { for (GroupStatus sched : scoreboard) { if (sched.group.id() == group.id()) { - sched.release(success, (double) searchTimeMs / 1000.0); + sched.release(success, searchTimeMs / 1000.0); break; } } @@ -134,6 +134,7 @@ public class LoadBalancer { } private static class RoundRobinScheduler implements GroupScheduler { + private int needle = 0; private final List<GroupStatus> scoreboard; @@ -204,6 +205,7 @@ public class LoadBalancer { } static class AdaptiveScheduler implements GroupScheduler { + private final Random random; private final List<GroupStatus> scoreboard; @@ -251,4 +253,5 @@ public class LoadBalancer { return selectGroup(needle, false, rejectedGroups); } } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java b/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java index c2e81d43677..3ebd21fa18a 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java @@ -9,5 +9,7 @@ package com.yahoo.search.dispatch; * @author ollivir */ public interface ResponseMonitor<T> { + void responseAvailable(T from); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java index a32931c43c8..7dbc2e98759 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java @@ -35,11 +35,12 @@ public class SearchErrorInvoker extends SearchInvoker { } @Override - protected void sendSearchRequest(Query query) throws IOException { + protected Object sendSearchRequest(Query query, Object context) throws IOException { this.query = query; - if(monitor != null) { + if (monitor != null) { monitor.responseAvailable(this); } + return context; } @Override @@ -60,4 +61,5 @@ public class SearchErrorInvoker extends SearchInvoker { protected void setMonitor(ResponseMonitor<SearchInvoker> monitor) { this.monitor = monitor; } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java index 45ed1b87746..b33e91189cc 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java @@ -32,14 +32,21 @@ public abstract class SearchInvoker extends CloseableInvoker { * for correct result windowing. */ public Result search(Query query, Execution execution) throws IOException { - sendSearchRequest(query); + sendSearchRequest(query, null); InvokerResult result = getSearchResult(execution); setFinalStatus(result.getResult().hits().getError() == null); result.complete(); return result.getResult(); } - protected abstract void sendSearchRequest(Query query) throws IOException; + /** + * + * @param query the query to send + * @param context a context object that can be used to pass context among different + * invokers, e.g for reuse of preserialized data. + * @return an object that can be passed to the next invocation of sendSearchRequest + */ + protected abstract Object sendSearchRequest(Query query, Object context) throws IOException; protected abstract InvokerResult getSearchResult(Execution execution) throws IOException; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java index 1e0153761c9..b29c3297aea 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; -import com.google.common.collect.ImmutableCollection; import com.yahoo.collections.Pair; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; @@ -13,6 +12,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Random; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -28,15 +28,11 @@ import java.util.stream.IntStream; public class SearchPath { /** - * Parse the search path and select nodes from the given cluster based on it. + * Parses the search path and select nodes from the given cluster based on it. * - * @param searchPath - * unparsed search path expression (see: model.searchPath in Search - * API reference) - * @param cluster - * the search cluster from which nodes are selected - * @throws InvalidSearchPathException - * if the searchPath is malformed + * @param searchPath unparsed search path expression (see: model.searchPath in Search API reference) + * @param cluster the search cluster from which nodes are selected + * @throws InvalidSearchPathException if the searchPath is malformed * @return list of nodes chosen with the search path, or an empty list in which * case some other node selection logic should be used */ @@ -45,7 +41,7 @@ public class SearchPath { if (sp.isPresent()) { return sp.get().mapToNodes(cluster); } else { - return Collections.emptyList(); + return List.of(); } } @@ -68,17 +64,18 @@ public class SearchPath { } } - private final List<NodeSelection> nodes; - private final Integer group; + private final List<Selection> nodes; + private final List<Selection> groups; + private static final Random random = new Random(); - private SearchPath(List<NodeSelection> nodes, Integer group) { + private SearchPath(List<Selection> nodes, List<Selection> groups) { this.nodes = nodes; - this.group = group; + this.groups = groups; } private List<Node> mapToNodes(SearchCluster cluster) { if (cluster.groups().isEmpty()) { - return Collections.emptyList(); + return List.of(); } Group selectedGroup = selectGroup(cluster); @@ -89,7 +86,7 @@ public class SearchPath { List<Node> groupNodes = selectedGroup.nodes(); Set<Integer> wanted = new HashSet<>(); int max = groupNodes.size(); - for (NodeSelection node : nodes) { + for (Selection node : nodes) { wanted.addAll(node.matches(max)); } List<Node> ret = new ArrayList<>(); @@ -100,41 +97,52 @@ public class SearchPath { } private boolean isEmpty() { - return nodes.isEmpty() && group == null; + return nodes.isEmpty() && groups.isEmpty(); } - private Group selectGroup(SearchCluster cluster) { - if (group != null) { - Optional<Group> specificGroup = cluster.group(group); - if (specificGroup.isPresent()) { - return specificGroup.get(); + private Group selectRandomGroupWithSufficientCoverage(SearchCluster cluster, List<Integer> groupIds) { + while ( groupIds.size() > 1 ) { + int index = random.nextInt(groupIds.size()); + int groupId = groupIds.get(index); + Optional<Group> group = cluster.group(groupId); + if (group.isPresent()) { + if (group.get().hasSufficientCoverage()) { + return group.get(); + } else { + groupIds.remove(index); + } } else { - throw new InvalidSearchPathException("Invalid searchPath, cluster does not have " + (group + 1) + " groups"); + throw new InvalidSearchPathException("Invalid searchPath, cluster does not have " + (groupId + 1) + " groups"); } } + return cluster.group(groupIds.get(0)).get(); + } - // pick "anything": try to find the first working - ImmutableCollection<Group> groups = cluster.groups().values(); - for (Group g : groups) { - if (g.hasSufficientCoverage()) { - return g; + private Group selectGroup(SearchCluster cluster) { + if ( ! groups.isEmpty()) { + List<Integer> potentialGroups = new ArrayList<>(); + for (Selection groupSelection : groups) { + for (int group = groupSelection.from; group < groupSelection.to; group++) { + potentialGroups.add(group); + } } + return selectRandomGroupWithSufficientCoverage(cluster, potentialGroups); } - // fallback: first - return groups.iterator().next(); + // pick any working group + return selectRandomGroupWithSufficientCoverage(cluster, new ArrayList<>(cluster.groups().keySet())); } private static SearchPath parseElement(String element) { - Pair<String, String> nodesAndGroup = halveAt('/', element); - List<NodeSelection> nodes = parseNodes(nodesAndGroup.getFirst()); - Integer group = parseGroup(nodesAndGroup.getSecond()); + Pair<String, String> nodesAndGroups = halveAt('/', element); + List<Selection> nodes = parseSelection(nodesAndGroups.getFirst()); + List<Selection> groups = parseSelection(nodesAndGroups.getSecond()); - return new SearchPath(nodes, group); + return new SearchPath(nodes, groups); } - private static List<NodeSelection> parseNodes(String nodes) { - List<NodeSelection> ret = new ArrayList<>(); + private static List<Selection> parseSelection(String nodes) { + List<Selection> ret = new ArrayList<>(); while (nodes.length() > 0) { if (nodes.startsWith("[")) { nodes = parseNodeRange(nodes, ret); @@ -148,8 +156,8 @@ public class SearchPath { return ret; } - // an asterisk or an empty string followed by a comma or the end of the string - private static final Pattern NODE_WILDCARD = Pattern.compile("^\\*?(?:,|$)"); + // An asterisk or forward slash or an empty string followed by a comma or the end of the string + private static final Pattern NODE_WILDCARD = Pattern.compile("^\\*?(?:,|$|/$)"); private static boolean isWildcard(String node) { return NODE_WILDCARD.matcher(node).lookingAt(); @@ -157,40 +165,30 @@ public class SearchPath { private static final Pattern NODE_RANGE = Pattern.compile("^\\[(\\d+),(\\d+)>(?:,|$)"); - private static String parseNodeRange(String nodes, List<NodeSelection> into) { + private static String parseNodeRange(String nodes, List<Selection> into) { Matcher m = NODE_RANGE.matcher(nodes); if (m.find()) { String ret = nodes.substring(m.end()); - Integer start = Integer.parseInt(m.group(1)); - Integer end = Integer.parseInt(m.group(2)); + int start = Integer.parseInt(m.group(1)); + int end = Integer.parseInt(m.group(2)); if (start > end) { throw new InvalidSearchPathException("Invalid range"); } - into.add(new NodeSelection(start, end)); + into.add(new Selection(start, end)); return ret; } else { throw new InvalidSearchPathException("Invalid range expression"); } } - private static String parseNodeNum(String nodes, List<NodeSelection> into) { + private static String parseNodeNum(String nodes, List<Selection> into) { Pair<String, String> numAndRest = halveAt(',', nodes); int nodeNum = Integer.parseInt(numAndRest.getFirst()); - into.add(new NodeSelection(nodeNum, nodeNum + 1)); + into.add(new Selection(nodeNum, nodeNum + 1)); return numAndRest.getSecond(); } - private static Integer parseGroup(String group) { - if (group.isEmpty()) { - return null; - } - if ("/".equals(group) || "*".equals(group)) { // anything goes - return null; - } - return Integer.parseInt(group); - } - private static Pair<String, String> halveAt(char divider, String string) { int pos = string.indexOf(divider); if (pos >= 0) { @@ -199,11 +197,9 @@ public class SearchPath { return new Pair<>(string, ""); } - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); + private static void selectionToString(StringBuilder sb, List<Selection> nodes) { boolean first = true; - for (NodeSelection p : nodes) { + for (Selection p : nodes) { if (first) { first = false; } else { @@ -211,17 +207,24 @@ public class SearchPath { } sb.append(p.toString()); } - if (group != null) { - sb.append('/').append(group); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + selectionToString(sb, nodes); + if ( ! groups.isEmpty()) { + sb.append('/'); + selectionToString(sb, groups); } return sb.toString(); } - private static class NodeSelection { + private static class Selection { private final int from; private final int to; - NodeSelection(int from, int to) { + Selection(int from, int to) { this.from = from; this.to = to; } @@ -230,7 +233,7 @@ public class SearchPath { if (from >= max) { return Collections.emptyList(); } - int end = (to > max) ? max : to; + int end = Math.min(to, max); return IntStream.range(from, end).boxed().collect(Collectors.toList()); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java new file mode 100644 index 00000000000..315dfdd4320 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java @@ -0,0 +1,79 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch; + +import org.apache.commons.math3.distribution.TDistribution; + +/** + * Use StudentT distribution and estimate how many hits you need from each partition + * to to get the globally top-k documents with the desired probability + * + * @author baldersheim + */ +public class TopKEstimator { + + private final TDistribution studentT; + private final double defaultP; + private final boolean estimate; + private final double skewFactor; + private final double [] defaultCumulativeProbability; + private final static int MIN_N = 2; + + private static boolean needEstimate(double p) { + return (0.0 < p) && (p < 1.0); + } + + TopKEstimator(double freedom, double defaultProbability) { + this(freedom, defaultProbability, 0.0); + } + + public TopKEstimator(double freedom, double defaultProbability, double skewFactor) { + this.studentT = new TDistribution(null, freedom); + defaultP = defaultProbability; + estimate = needEstimate(defaultP); + this.skewFactor = skewFactor; + defaultCumulativeProbability = new double[64]; + for (int i=0; i < defaultCumulativeProbability.length; i++) { + defaultCumulativeProbability[i] = computeCumulativeProbability(i+MIN_N, defaultP); + } + } + + private double inverseCumulativeProbability(int n, double p) { + if (p == defaultP && (n >= MIN_N) && (n < defaultCumulativeProbability.length + MIN_N)) { + return defaultCumulativeProbability[n - MIN_N]; + } + return computeCumulativeProbability(n, p); + } + + private double computeCumulativeProbability(int n, double p) { + double p_inverse = 1 - (1 - p)/computeN(n); + return studentT.inverseCumulativeProbability(p_inverse); + } + + private double computeN(double n) { + double p_max = (1 + skewFactor)/n; + return Math.max(1, 1/p_max); + } + + double estimateExactK(double k, int n_i, double p) { + double n = computeN(n_i); + double variance = k * 1/n * (1 - 1/n); + return k/n + inverseCumulativeProbability(n_i, p) * Math.sqrt(variance); + } + + double estimateExactK(double k, int n) { + return estimateExactK(k, n, defaultP); + } + + public int estimateK(int k, int n) { + return (estimate && (n >= MIN_N)) + ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, defaultP))) + : k; + } + + public int estimateK(int k, int n, double p) { + return (needEstimate(p) && (n >= MIN_N)) + ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, p))) + : k; + } +} + diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java index bc0a38617ee..53cbee114b9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java @@ -16,6 +16,7 @@ interface Client { /** Creates a connection to a particular node in this */ NodeConnection createConnection(String hostname, int port); + void close(); interface ResponseReceiver { void receive(ResponseOrError<ProtobufResponse> response); @@ -87,7 +88,7 @@ interface Client { RpcFillInvoker.GetDocsumsResponseReceiver responseReceiver, double timeoutSeconds); void request(String rpcMethod, CompressionType compression, int uncompressedLength, byte[] compressedPayload, - ResponseReceiver responseReceiver, double timeoutSeconds); + ResponseReceiver responseReceiver, double timeoutSeconds); /** Closes this connection */ void close(); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index ae2258c4546..250524fadf2 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -101,7 +101,7 @@ public class ProtobufSerialization { mergeToSearchRequestFromSorting(ranking.getSorting(), builder); } if (ranking.getLocation() != null) { - builder.setGeoLocation(ranking.getLocation().toString()); + builder.setGeoLocation(ranking.getLocation().backendString()); } var featureMap = ranking.getFeatures().asMap(); @@ -144,7 +144,7 @@ public class ProtobufSerialization { builder.setRankProfile(ranking.getProfile()); if (ranking.getLocation() != null) { - builder.setGeoLocation(ranking.getLocation().toString()); + builder.setGeoLocation(ranking.getLocation().backendString()); } if (includeQueryData) { mergeQueryDataToDocsumRequest(query, builder); @@ -196,7 +196,7 @@ public class ProtobufSerialization { result.getResult().setTotalHitCount(protobuf.getTotalHitCount()); result.getResult().setCoverage(convertToCoverage(protobuf)); - var haveGrouping = protobuf.getGroupingBlob() != null && !protobuf.getGroupingBlob().isEmpty(); + var haveGrouping = ! protobuf.getGroupingBlob().isEmpty(); if (haveGrouping) { BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer(protobuf.getGroupingBlob().asReadOnlyByteBuffer())); int cnt = buf.getInt(null); @@ -214,12 +214,12 @@ public class ProtobufSerialization { for (var replyHit : protobuf.getHitsList()) { LeanHit hit = (replyHit.getSortData().isEmpty()) ? new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getRelevance()) - : new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getSortData().toByteArray()); + : new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getRelevance(), replyHit.getSortData().toByteArray()); result.getLeanHits().add(hit); } var slimeTrace = protobuf.getSlimeTrace(); - if (slimeTrace != null && !slimeTrace.isEmpty()) { + if ( ! slimeTrace.isEmpty()) { var traces = new Value.ArrayValue(); traces.add(new SlimeAdapter(BinaryFormat.decode(slimeTrace.toByteArray()).get())); query.trace(traces, query.getTraceLevel()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java index a2821892358..70d94c5a8a6 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java @@ -25,8 +25,13 @@ class RpcClient implements Client { private final Supervisor supervisor; - public RpcClient(int transportThreads) { - supervisor = new Supervisor(new Transport(transportThreads)); + public RpcClient(String name, int transportThreads) { + supervisor = new Supervisor(new Transport(name, transportThreads)); + } + + @Override + public void close() { + supervisor.transport().shutdown().join(); } @Override @@ -44,13 +49,14 @@ class RpcClient implements Client { // The current shared connection. This will be recycled when it becomes invalid. // All access to this must be synchronized - private Target target = null; + private Target target; public RpcNodeConnection(String hostname, int port, Supervisor supervisor) { this.supervisor = supervisor; this.hostname = hostname; this.port = port; description = "rpc node connection to " + hostname + ":" + port; + target = supervisor.connect(new Spec(hostname, port)); } @Override @@ -79,17 +85,16 @@ class RpcClient implements Client { private void invokeAsync(Request req, double timeout, RequestWaiter waiter) { // TODO: Consider replacing this by a watcher on the target synchronized(this) { // ensure we have exactly 1 valid connection across threads - if (target == null || ! target.isValid()) + if (! target.isValid()) { target = supervisor.connect(new Spec(hostname, port)); + } } target.invokeAsync(req, timeout, waiter); } @Override public void close() { - if (target != null) { - target.close(); - } + target.close(); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java index 9b661368972..0e8759f740e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java @@ -137,7 +137,7 @@ public class RpcFillInvoker extends FillInvoker { root.setString("ranking", rankProfile); } if (location != null) { - root.setString("location", location.toString()); + root.setString("location", location.backendString()); } Cursor gids = root.setArray("gids"); for (FastHit hit : hits) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java index 5e04f1d7a3e..26abe92a6f1 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -22,6 +22,7 @@ public class RpcPing implements Pinger, Client.ResponseReceiver { private static final Logger log = Logger.getLogger(RpcPing.class.getName()); private static final String RPC_METHOD = "vespa.searchprotocol.ping"; private static final CompressionType PING_COMPRESSION = CompressionType.NONE; + private static final boolean triggeredClassLoading = ErrorMessage.createBackendCommunicationError("TriggerClassLoading") instanceof ErrorMessage; private final Node node; private final RpcResourcePool resourcePool; @@ -86,6 +87,7 @@ public class RpcPing implements Pinger, Client.ResponseReceiver { @Override public void receive(ResponseOrError<ProtobufResponse> response) { + if (clusterMonitor.isClosed() && ! triggeredClassLoading) return; if (node.isLastReceivedPong(pingSequenceId)) { pongHandler.handle(toPong(response)); } else { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java index ac8f0a59c20..01e3ec3ca2b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; import com.yahoo.search.cluster.ClusterMonitor; @@ -7,12 +8,16 @@ import com.yahoo.search.dispatch.searchcluster.Pinger; import com.yahoo.search.dispatch.searchcluster.PongHandler; public class RpcPingFactory implements PingFactory { + private final RpcResourcePool rpcResourcePool; + public RpcPingFactory(RpcResourcePool rpcResourcePool) { this.rpcResourcePool = rpcResourcePool; } + @Override public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) { return new RpcPing(node, monitor, rpcResourcePool, pongHandler); } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java index 341b9b2bce3..8a17be8102e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java @@ -38,6 +38,7 @@ import java.util.logging.Logger; * @author ollivir */ public class RpcProtobufFillInvoker extends FillInvoker { + private static final String RPC_METHOD = "vespa.searchprotocol.getDocsums"; private static final Logger log = Logger.getLogger(RpcProtobufFillInvoker.class.getName()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java index 065489ef9a0..c3d072b8db6 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java @@ -4,7 +4,6 @@ package com.yahoo.search.dispatch.rpc; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.component.ComponentId; import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.compress.Compressor.Compression; @@ -27,6 +26,7 @@ import java.util.Random; * @author ollivir */ public class RpcResourcePool extends AbstractComponent { + /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ public final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); @@ -35,17 +35,19 @@ public class RpcResourcePool extends AbstractComponent { /** Connections to the search nodes this talks to, indexed by node id ("partid") */ private final ImmutableMap<Integer, NodeConnectionPool> nodeConnectionPools; + private final RpcClient client; RpcResourcePool(Map<Integer, NodeConnection> nodeConnections) { var builder = new ImmutableMap.Builder<Integer, NodeConnectionPool>(); nodeConnections.forEach((key, connection) -> builder.put(key, new NodeConnectionPool(Collections.singletonList(connection)))); this.nodeConnectionPools = builder.build(); + client = null; } @Inject public RpcResourcePool(DispatchConfig dispatchConfig) { super(); - var client = new RpcClient(dispatchConfig.numJrtTransportThreads()); + client = new RpcClient("dispatch-client", dispatchConfig.numJrtTransportThreads()); // Create rpc node connection pools indexed by the node distribution key var builder = new ImmutableMap.Builder<Integer, NodeConnectionPool>(); @@ -82,6 +84,9 @@ public class RpcResourcePool extends AbstractComponent { public void deconstruct() { super.deconstruct(); nodeConnectionPools.values().forEach(NodeConnectionPool::release); + if (client != null) { + client.close(); + } } private class NodeConnectionPool { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 76240e55c98..20b11efb470 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -45,21 +45,36 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe } @Override - protected void sendSearchRequest(Query query) { + protected Object sendSearchRequest(Query query, Object incomingContext) { this.query = query; Client.NodeConnection nodeConnection = resourcePool.getConnection(node.key()); if (nodeConnection == null) { responses.add(Client.ResponseOrError.fromError("Could not send search to unknown node " + node.key())); responseAvailable(); - return; + return incomingContext; } query.trace(false, 5, "Sending search request with jrt/protobuf to node with dist key ", node.key()); - var payload = ProtobufSerialization.serializeSearchRequest(query, Math.min(query.getHits(), maxHits), searcher.getServerId()); + RpcContext context = getContext(incomingContext); double timeoutSeconds = ((double) query.getTimeLeft() - 3.0) / 1000.0; - Compressor.Compression compressionResult = resourcePool.compress(query, payload); - nodeConnection.request(RPC_METHOD, compressionResult.type(), payload.length, compressionResult.data(), this, timeoutSeconds); + nodeConnection.request(RPC_METHOD, + context.compressedPayload.type(), + context.compressedPayload.uncompressedSize(), + context.compressedPayload.data(), + this, + timeoutSeconds); + return context; + } + + private RpcContext getContext(Object incomingContext) { + if (incomingContext instanceof RpcContext) + return (RpcContext)incomingContext; + + return new RpcContext(resourcePool, query, + ProtobufSerialization.serializeSearchRequest(query, + Math.min(query.getHits(), maxHits), + searcher.getServerId())); } @Override @@ -87,9 +102,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe ProtobufResponse protobufResponse = response.response().get(); CompressionType compression = CompressionType.valueOf(protobufResponse.compression()); byte[] payload = resourcePool.compressor().decompress(protobufResponse.compressedPayload(), compression, protobufResponse.uncompressedSize()); - var result = ProtobufSerialization.deserializeToSearchResult(payload, query, searcher, node.pathIndex(), node.key()); - - return result; + return ProtobufSerialization.deserializeToSearchResult(payload, query, searcher, node.pathIndex(), node.key()); } @Override @@ -106,4 +119,14 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe return searcher.getName(); } + static class RpcContext { + + final Compressor.Compression compressedPayload; + + RpcContext(RpcResourcePool resourcePool, Query query, byte[] payload) { + compressedPayload = resourcePool.compress(query, payload); + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java index 0e4e87b9a6a..727fb64faef 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java @@ -6,6 +6,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Logger; /** * A group in a search cluster. This class is multithread safe. @@ -15,12 +16,17 @@ import java.util.concurrent.atomic.AtomicLong; */ public class Group { + private static final Logger log = Logger.getLogger(Group.class.getName()); + private final static double maxContentSkew = 0.10; // If documents on a node is more than 10% off from the average the group is unbalanced + private final static int minDocsPerNodeToRequireLowSkew = 100; + private final int id; private final ImmutableList<Node> nodes; - private final AtomicBoolean hasSufficientCoverage = new AtomicBoolean(true); private final AtomicBoolean hasFullCoverage = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); + private final AtomicBoolean isBlockingWrites = new AtomicBoolean(false); + private final AtomicBoolean isBalanced = new AtomicBoolean(true); public Group(int id, List<Node> nodes) { this.id = id; @@ -52,38 +58,52 @@ public class Group { } public int workingNodes() { - int nodesUp = 0; - for (Node node : nodes) { - if (node.isWorking() == Boolean.TRUE) { - nodesUp++; - } - } - return nodesUp; + return (int) nodes.stream().filter(node -> node.isWorking() == Boolean.TRUE).count(); } - void aggregateActiveDocuments() { - long activeDocumentsInGroup = 0; - for (Node node : nodes) { - if (node.isWorking() == Boolean.TRUE) { - activeDocumentsInGroup += node.getActiveDocuments(); + public void aggregateNodeValues() { + long activeDocs = nodes.stream().filter(node -> node.isWorking() == Boolean.TRUE).mapToLong(Node::getActiveDocuments).sum(); + activeDocuments.set(activeDocs); + isBlockingWrites.set(nodes.stream().anyMatch(Node::isBlockingWrites)); + int numWorkingNodes = workingNodes(); + if (numWorkingNodes > 0) { + long average = activeDocs / numWorkingNodes; + long skew = nodes.stream().filter(node -> node.isWorking() == Boolean.TRUE).mapToLong(node -> Math.abs(node.getActiveDocuments() - average)).sum(); + boolean balanced = skew <= activeDocs * maxContentSkew; + if (!isBalanced.get() || balanced != isBalanced.get()) { + if (!isSparse()) + log.info("Content is " + (balanced ? "" : "not ") + "well balanced. Current deviation = " + + skew * 100 / activeDocs + " %. activeDocs = " + activeDocs + ", skew = " + skew + + ", average = " + average); + isBalanced.set(balanced); } + } else { + isBalanced.set(true); } - activeDocuments.set(activeDocumentsInGroup); - } - /** Returns the active documents on this node. If unknown, 0 is returned. */ - long getActiveDocuments() { - return this.activeDocuments.get(); + /** Returns the active documents on this group. If unknown, 0 is returned. */ + long activeDocuments() { return activeDocuments.get(); } + + /** Returns whether any node in this group is currently blocking write operations */ + public boolean isBlockingWrites() { return isBlockingWrites.get(); } + + /** Returns whether the nodes in the group have about the same number of documents */ + public boolean isBalanced() { return isBalanced.get(); } + + /** Returns whether this group has too few documents per node to expect it to be balanced */ + public boolean isSparse() { + if (nodes.isEmpty()) return false; + return activeDocuments.get() / nodes.size() < minDocsPerNodeToRequireLowSkew; } - public boolean isFullCoverageStatusChanged(boolean hasFullCoverageNow) { + public boolean fullCoverageStatusChanged(boolean hasFullCoverageNow) { boolean previousState = hasFullCoverage.getAndSet(hasFullCoverageNow); return previousState != hasFullCoverageNow; } @Override - public String toString() { return "search group " + id; } + public String toString() { return "group " + id; } @Override public int hashCode() { return id; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index e93b633f09d..9807a978647 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -23,6 +23,7 @@ public class Node { private final AtomicLong activeDocuments = new AtomicLong(0); private final AtomicLong pingSequence = new AtomicLong(0); private final AtomicLong lastPong = new AtomicLong(0); + private final AtomicBoolean isBlockingWrites = new AtomicBoolean(false); public Node(int key, String hostname, int group) { this.key = key; @@ -70,14 +71,14 @@ public class Node { } /** Updates the active documents on this node */ - void setActiveDocuments(long activeDocuments) { - this.activeDocuments.set(activeDocuments); - } + public void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); } /** Returns the active documents on this node. If unknown, 0 is returned. */ - long getActiveDocuments() { - return activeDocuments.get(); - } + long getActiveDocuments() { return activeDocuments.get(); } + + public void setBlockingWrites(boolean isBlockingWrites) { this.isBlockingWrites.set(isBlockingWrites); } + + boolean isBlockingWrites() { return isBlockingWrites.get(); } @Override public int hashCode() { return Objects.hash(hostname, key, pathIndex, group); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java index 2e07d8d61e6..3b9e9573367 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java @@ -3,7 +3,9 @@ package com.yahoo.search.dispatch.searchcluster; import com.yahoo.search.cluster.ClusterMonitor; - +/** + * @author ollivir + */ public interface PingFactory { Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java index b4a7ccbf98c..681a7d0af2c 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java @@ -8,5 +8,7 @@ package com.yahoo.search.dispatch.searchcluster; * @author baldersheim */ public interface Pinger { + void ping(); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java index c0579b5d36e..c39426e9d76 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.searchcluster; import com.yahoo.prelude.Pong; @@ -8,5 +9,7 @@ import com.yahoo.prelude.Pong; * @author baldersheim */ public interface PongHandler { + void handle(Pong pong); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 06e3934335a..b5fbede4701 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -5,18 +5,19 @@ import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; +import com.google.common.math.Quantiles; import com.yahoo.container.handler.VipStatus; import com.yahoo.net.HostName; import com.yahoo.prelude.Pong; import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.cluster.NodeManager; +import com.yahoo.search.dispatch.TopKEstimator; import com.yahoo.vespa.config.search.DispatchConfig; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; import java.util.concurrent.Executor; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -38,7 +39,9 @@ public class SearchCluster implements NodeManager<Node> { private final ImmutableList<Group> orderedGroups; private final VipStatus vipStatus; private final PingFactory pingFactory; + private final TopKEstimator hitEstimator; private long nextLogTime = 0; + private static final double SKEW_FACTOR = 0.05; /** * A search node on this local machine having the entire corpus, which we therefore @@ -76,12 +79,13 @@ public class SearchCluster implements NodeManager<Node> { for (Node node : nodes) nodesByHostBuilder.put(node.hostname(), node); this.nodesByHost = nodesByHostBuilder.build(); + hitEstimator = new TopKEstimator(30.0, dispatchConfig.topKProbability(), SKEW_FACTOR); this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), nodesByHost, groups); } - public void addMonitoring(ClusterMonitor clusterMonitor) { - for (var group : orderedGroups) { + public void addMonitoring(ClusterMonitor<Node> clusterMonitor) { + for (var group : orderedGroups()) { for (var node : group.nodes()) clusterMonitor.add(node, true); } @@ -129,22 +133,25 @@ public class SearchCluster implements NodeManager<Node> { /** Returns the n'th (zero-indexed) group in the cluster if possible */ public Optional<Group> group(int n) { - if (orderedGroups.size() > n) { - return Optional.of(orderedGroups.get(n)); + if (orderedGroups().size() > n) { + return Optional.of(orderedGroups().get(n)); } else { return Optional.empty(); } } - /** Returns the number of nodes per group - size()/groups.size() */ - public int groupSize() { - if (groups.size() == 0) return size(); - return size() / groups.size(); + /** + * Returns the wanted number of nodes per group - size()/groups.size(). + * The actual node count for a given group may differ due to node retirements. + */ + public int wantedGroupSize() { + if (groups().size() == 0) return size(); + return size() / groups().size(); } public int groupsWithSufficientCoverage() { int covered = 0; - for (Group g : orderedGroups) { + for (Group g : orderedGroups()) { if (g.hasSufficientCoverage()) { covered++; } @@ -160,7 +167,7 @@ public class SearchCluster implements NodeManager<Node> { if ( localCorpusDispatchTarget.isEmpty()) return Optional.empty(); // Only use direct dispatch if the local group has sufficient coverage - Group localSearchGroup = groups.get(localCorpusDispatchTarget.get().group()); + Group localSearchGroup = groups().get(localCorpusDispatchTarget.get().group()); if ( ! localSearchGroup.hasSufficientCoverage()) return Optional.empty(); // Only use direct dispatch if the local search node is not down @@ -199,7 +206,10 @@ public class SearchCluster implements NodeManager<Node> { setInRotationOnlyIf(hasWorkingNodes()); } else if (usesLocalCorpusIn(node)) { // follow the status of this node - setInRotationOnlyIf(nodeIsWorking); + // Do not take this out of rotation if we're a combined cluster of size 1, + // as that can't be helpful, and leads to a deadlock where this node is never taken back in servic e + if (nodeIsWorking || size() > 1) + setInRotationOnlyIf(nodeIsWorking); } } @@ -219,6 +229,13 @@ public class SearchCluster implements NodeManager<Node> { vipStatus.removeFromRotation(clusterId); } + public int estimateHitsToFetch(int wantedHits, int numPartitions) { + return hitEstimator.estimateK(wantedHits, numPartitions); + } + public int estimateHitsToFetch(int wantedHits, int numPartitions, double topKProbability) { + return hitEstimator.estimateK(wantedHits, numPartitions, topKProbability); + } + public boolean hasInformationAboutAllNodes() { return nodesByHost.values().stream().allMatch(node -> node.isWorking() != null); } @@ -236,12 +253,15 @@ public class SearchCluster implements NodeManager<Node> { } private static class PongCallback implements PongHandler { + private final ClusterMonitor<Node> clusterMonitor; private final Node node; + PongCallback(Node node, ClusterMonitor<Node> clusterMonitor) { this.node = node; this.clusterMonitor = clusterMonitor; } + @Override public void handle(Pong pong) { if (pong.badResponse()) { @@ -249,10 +269,12 @@ public class SearchCluster implements NodeManager<Node> { } else { if (pong.activeDocuments().isPresent()) { node.setActiveDocuments(pong.activeDocuments().get()); + node.setBlockingWrites(pong.isBlockingWrites()); } clusterMonitor.responded(node); } } + } /** Used by the cluster monitor to manage node status */ @@ -263,41 +285,35 @@ public class SearchCluster implements NodeManager<Node> { } private void pingIterationCompletedSingleGroup() { - Group group = groups.values().iterator().next(); - group.aggregateActiveDocuments(); + Group group = groups().values().iterator().next(); + group.aggregateNodeValues(); // With just one group sufficient coverage may not be the same as full coverage, as the // group will always be marked sufficient for use. updateSufficientCoverage(group, true); - boolean fullCoverage = isGroupCoverageSufficient(group.workingNodes(), group.nodes().size(), group.getActiveDocuments(), - group.getActiveDocuments()); - trackGroupCoverageChanges(0, group, fullCoverage, group.getActiveDocuments()); + boolean sufficientCoverage = isGroupCoverageSufficient(group.activeDocuments(), + group.activeDocuments()); + trackGroupCoverageChanges(group, sufficientCoverage, group.activeDocuments()); } private void pingIterationCompletedMultipleGroups() { - int numGroups = orderedGroups.size(); - // Update active documents per group and use it to decide if the group should be active - - long[] activeDocumentsInGroup = new long[numGroups]; - long sumOfActiveDocuments = 0; - for(int i = 0; i < numGroups; i++) { - Group group = orderedGroups.get(i); - group.aggregateActiveDocuments(); - activeDocumentsInGroup[i] = group.getActiveDocuments(); - sumOfActiveDocuments += activeDocumentsInGroup[i]; - } - + orderedGroups().forEach(Group::aggregateNodeValues); + long medianDocuments = medianDocumentsPerGroup(); boolean anyGroupsSufficientCoverage = false; - for (int i = 0; i < numGroups; i++) { - Group group = orderedGroups.get(i); - long activeDocuments = activeDocumentsInGroup[i]; - long averageDocumentsInOtherGroups = (sumOfActiveDocuments - activeDocuments) / (numGroups - 1); - boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), group.nodes().size(), activeDocuments, averageDocumentsInOtherGroups); + for (Group group : orderedGroups()) { + boolean sufficientCoverage = isGroupCoverageSufficient(group.activeDocuments(), + medianDocuments); anyGroupsSufficientCoverage = anyGroupsSufficientCoverage || sufficientCoverage; updateSufficientCoverage(group, sufficientCoverage); - trackGroupCoverageChanges(i, group, sufficientCoverage, averageDocumentsInOtherGroups); + trackGroupCoverageChanges(group, sufficientCoverage, medianDocuments); } } + private long medianDocumentsPerGroup() { + if (orderedGroups().isEmpty()) return 0; + var activeDocuments = orderedGroups().stream().map(Group::activeDocuments).collect(Collectors.toList()); + return (long)Quantiles.median().compute(activeDocuments); + } + /** * Update statistics after a round of issuing pings. * Note that this doesn't wait for pings to return, so it will typically accumulate data from @@ -305,7 +321,7 @@ public class SearchCluster implements NodeManager<Node> { */ @Override public void pingIterationCompleted() { - int numGroups = orderedGroups.size(); + int numGroups = orderedGroups().size(); if (numGroups == 1) { pingIterationCompletedSingleGroup(); } else { @@ -313,77 +329,42 @@ public class SearchCluster implements NodeManager<Node> { } } - private boolean isGroupCoverageSufficient(int workingNodes, int nodesInGroup, long activeDocuments, long averageDocumentsInOtherGroups) { - boolean sufficientCoverage = true; - - if (averageDocumentsInOtherGroups > 0) { - double coverage = 100.0 * (double) activeDocuments / averageDocumentsInOtherGroups; - sufficientCoverage = coverage >= dispatchConfig.minActivedocsPercentage(); - } - if (sufficientCoverage) { - sufficientCoverage = isGroupNodeCoverageSufficient(workingNodes, nodesInGroup); - } - return sufficientCoverage; - } - - private boolean isGroupNodeCoverageSufficient(int workingNodes, int nodesInGroup) { - int nodesAllowedDown = dispatchConfig.maxNodesDownPerGroup() - + (int) (((double) nodesInGroup * (100.0 - dispatchConfig.minGroupCoverage())) / 100.0); - return workingNodes + nodesAllowedDown >= nodesInGroup; + private boolean isGroupCoverageSufficient(long activeDocuments, long medianDocuments) { + double documentCoverage = 100.0 * (double) activeDocuments / medianDocuments; + if (medianDocuments > 0 && documentCoverage < dispatchConfig.minActivedocsPercentage()) + return false; + return true; } /** * Calculate whether a subset of nodes in a group has enough coverage */ - public boolean isPartialGroupCoverageSufficient(OptionalInt knownGroupId, List<Node> nodes) { - if (orderedGroups.size() == 1) { - boolean sufficient = nodes.size() >= groupSize() - dispatchConfig.maxNodesDownPerGroup(); - return sufficient; - } - - if (knownGroupId.isEmpty()) { - return false; - } - int groupId = knownGroupId.getAsInt(); - Group group = groups.get(groupId); - if (group == null) { - return false; - } - int nodesInGroup = group.nodes().size(); - long sumOfActiveDocuments = 0; - int otherGroups = 0; - for (Group g : orderedGroups) { - if (g.id() != groupId) { - sumOfActiveDocuments += g.getActiveDocuments(); - otherGroups++; - } - } - long activeDocuments = 0; - for (Node n : nodes) { - activeDocuments += n.getActiveDocuments(); - } - long averageDocumentsInOtherGroups = sumOfActiveDocuments / otherGroups; - return isGroupCoverageSufficient(nodes.size(), nodesInGroup, activeDocuments, averageDocumentsInOtherGroups); + public boolean isPartialGroupCoverageSufficient(List<Node> nodes) { + if (orderedGroups().size() == 1) + return true; + long activeDocuments = nodes.stream().mapToLong(Node::getActiveDocuments).sum(); + return isGroupCoverageSufficient(activeDocuments, medianDocumentsPerGroup()); } - private void trackGroupCoverageChanges(int index, Group group, boolean fullCoverage, long averageDocuments) { + private void trackGroupCoverageChanges(Group group, boolean fullCoverage, long medianDocuments) { if ( ! hasInformationAboutAllNodes()) return; // Be silent until we know what we are talking about. - boolean changed = group.isFullCoverageStatusChanged(fullCoverage); + boolean changed = group.fullCoverageStatusChanged(fullCoverage); if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { nextLogTime = System.currentTimeMillis() + 30 * 1000; - int requiredNodes = groupSize() - dispatchConfig.maxNodesDownPerGroup(); if (fullCoverage) { - log.info(() -> String.format("Group %d is now good again (%d/%d active docs, coverage %d/%d)", - index, group.getActiveDocuments(), averageDocuments, group.workingNodes(), groupSize())); + log.info("Cluster " + clusterId + ": " + group + " has full coverage. " + + "Active documents: " + group.activeDocuments() + "/" + medianDocuments + ", " + + "working nodes: " + group.workingNodes() + "/" + group.nodes().size()); } else { - StringBuilder missing = new StringBuilder(); + StringBuilder unresponsive = new StringBuilder(); for (var node : group.nodes()) { - if (node.isWorking() != Boolean.TRUE) { - missing.append('\n').append(node.toString()); - } + if (node.isWorking() != Boolean.TRUE) + unresponsive.append('\n').append(node); } - log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d) (%d/%d active docs) Failed nodes are:%s", - index, group.workingNodes(), groupSize(), requiredNodes, group.getActiveDocuments(), averageDocuments, missing.toString())); + log.warning("Cluster " + clusterId + ": " + group + " has reduced coverage: " + + "Active documents: " + group.activeDocuments() + "/" + medianDocuments + ", " + + "working nodes: " + group.workingNodes() + "/" + group.nodes().size() + + ", unresponsive nodes: " + (unresponsive.toString().isEmpty() ? " none" : unresponsive)); } } } diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java b/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java index 5f1cfccf549..6243dc694c2 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java @@ -39,8 +39,8 @@ class FederationResult { } /** - * Wait on each target for that targets timeout - * On the worst case this is the same as waiting for the max target timeout, + * Wait on each target for that targets timeout. + * In the worst case this is the same as waiting for the max target timeout, * in the average case it may be much better because lower timeout sources do not get to * drive the timeout above their own timeout value. * When this completes, results can be accessed from the TargetResults with no blocking diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index d0b3189a79c..8d0e4944ab8 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -14,6 +14,7 @@ import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.errorhandling.Results; import com.yahoo.errorhandling.Results.Builder; import com.yahoo.prelude.IndexFacts; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -57,7 +58,6 @@ import java.util.logging.Logger; import static com.google.common.base.Preconditions.checkNotNull; import static com.yahoo.collections.CollectionUtil.first; -import static com.yahoo.container.util.Util.quote; import static com.yahoo.search.federation.StrictContractsConfig.PropagateSourceProperties; /** @@ -78,7 +78,6 @@ public class FederationSearcher extends ForkingSearcher { /** The name of the query property containing the source name added to the query to each source by this */ public final static CompoundName SOURCENAME = new CompoundName("sourceName"); public final static CompoundName PROVIDERNAME = new CompoundName("providerName"); - /** Logging field name constants */ public static final String LOG_COUNT_PREFIX = "count_"; @@ -105,7 +104,7 @@ public class FederationSearcher extends ForkingSearcher { ComponentRegistry<TargetSelector> targetSelectors) { if (selectorId.isEmpty()) return null; return checkNotNull(targetSelectors.getComponent(selectorId), - "Missing target selector with id" + quote(selectorId)); + "Missing target selector with id '" + selectorId + "'"); } // for testing @@ -139,7 +138,7 @@ public class FederationSearcher extends ForkingSearcher { } } - //Allow source groups to use by default. + // Allow source groups to use by default. if (target.useByDefault()) builder.useTargetByDefault(target.id()); } @@ -304,7 +303,6 @@ public class FederationSearcher extends ForkingSearcher { Object value = getSourceOrProviderProperty(original, key, sourceName, providerName, window.get(key)); if (value != null) outgoing.properties().set(key, value); - if (value != null) System.out.println("Setting " + key + " = " + value); } } } @@ -340,9 +338,8 @@ public class FederationSearcher extends ForkingSearcher { private List<String> allSourceRefDescriptions() { List<String> descriptions = new ArrayList<>(); - for (com.yahoo.search.federation.sourceref.Target target : searchChainResolver.allTopLevelTargets()) { + for (com.yahoo.search.federation.sourceref.Target target : searchChainResolver.allTopLevelTargets()) descriptions.add(target.searchRefDescription()); - } return descriptions; } @@ -355,7 +352,7 @@ public class FederationSearcher extends ForkingSearcher { } private void warnIfUnresolvedSearchChains(List<UnresolvedSearchChainException> missingTargets, - HitGroup errorHitGroup) { + HitGroup errorHitGroup) { if (!missingTargets.isEmpty()) { errorHitGroup.addError(missingSearchChainsErrorMessage(missingTargets)); } @@ -493,9 +490,9 @@ public class FederationSearcher extends ForkingSearcher { * TODO This is probably a dirty hack for bug 4711376. There are probably better ways. * But I will leave that to trd-processing@ * - * @param group The merging hitgroup to be updated if necessary - * @param orderer The per provider hit orderer. - * @return The hitorderer chosen + * @param group the merging hitgroup to be updated if necessary + * @param orderer the per provider hit orderer + * @return he hitorderer chosen */ private HitOrderer dirtyCopyIfModifiedOrderer(HitGroup group, HitOrderer orderer) { if (orderer != null) { @@ -546,8 +543,8 @@ public class FederationSearcher extends ForkingSearcher { private ComponentSpecification asSourceSpec(String source) { try { return new ComponentSpecification(source); - } catch(Exception e) { - throw new IllegalArgumentException("The source ref '" + source + "' used for federation is not valid.", e); + } catch (Exception e) { + throw new IllegalInputException("The source ref '" + source + "' used for federation is not valid.", e); } } diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainInvocationSpec.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainInvocationSpec.java index 6cb8d2ef174..59b4e521a56 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainInvocationSpec.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainInvocationSpec.java @@ -9,7 +9,7 @@ import java.util.List; import java.util.Objects; /** - * Specifices which search chain should be run and how it should be run. + * Specifies which search chain should be run and how it should be run. * This is a value object. * * @author Tony Vaagenes diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java index 97ceee96dfc..6626c1b3cc4 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java @@ -46,14 +46,12 @@ public class SearchChainResolver { public static class Builder { - private SortedSet<Target> defaultTargets = new TreeSet<>(); + private final SortedSet<Target> defaultTargets = new TreeSet<>(); - private final ComponentRegistry<Target> targets = new ComponentRegistry<Target>() { + private final ComponentRegistry<Target> targets = new ComponentRegistry<>() { @Override public void freeze() { - for (Target target : allComponents()) { - target.freeze(); - } + allComponents().forEach(Target::freeze); super.freeze(); } }; @@ -70,10 +68,16 @@ public class SearchChainResolver { return addSearchChain(searchChainId, new FederationOptions(), documentTypes); } - public Builder addSearchChain(ComponentId searchChainId, FederationOptions federationOptions, + public Builder addSearchChain(ComponentId searchChainId, + FederationOptions federationOptions, List<String> documentTypes) { registerTarget(new SingleTarget(searchChainId, - new SearchChainInvocationSpec(searchChainId, null, null, federationOptions, documentTypes), false)); + new SearchChainInvocationSpec(searchChainId, + null, + null, + federationOptions, + documentTypes), + false)); return this; } @@ -86,9 +90,8 @@ public class SearchChainResolver { } public Builder addSourceForProvider(ComponentId sourceId, ComponentId providerId, ComponentId searchChainId, - boolean isDefaultProviderForSource, FederationOptions federationOptions, - List<String> documentTypes) { - + boolean isDefaultProviderForSource, FederationOptions federationOptions, + List<String> documentTypes) { SearchChainInvocationSpec searchChainInvocationSpec = new SearchChainInvocationSpec(searchChainId, sourceId, providerId, federationOptions, documentTypes); @@ -129,7 +132,6 @@ public class SearchChainResolver { this.defaultTargets = Collections.unmodifiableSortedSet(defaultTargets); } - public SearchChainInvocationSpec resolve(ComponentSpecification sourceRef, Properties sourceToProviderMap) throws UnresolvedSearchChainException { @@ -158,4 +160,5 @@ public class SearchChainResolver { public SortedSet<Target> defaultTargets() { return defaultTargets; } + } diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java index 9c7e1024518..4613c73c4b4 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java @@ -5,8 +5,6 @@ import com.yahoo.component.ComponentId; import com.yahoo.processing.request.Properties; /** - * TODO: What is this? - * * @author Tony Vaagenes */ public class SingleTarget extends Target { diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java index ee98d033440..516ee9f968f 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java @@ -1,17 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.federation.sourceref; -import static com.yahoo.container.util.Util.quote; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.processing.request.Properties; import java.util.Arrays; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; -import com.yahoo.component.ComponentSpecification; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.processing.request.Properties; - /** * Maps a source reference to search chain invocation specs. * @@ -24,13 +22,12 @@ public class SourceRefResolver { public SourceRefResolver(SearchChainResolver searchChainResolver) { this.searchChainResolver = searchChainResolver; } + public Set<SearchChainInvocationSpec> resolve(ComponentSpecification sourceRef, Properties sourceToProviderMap, - IndexFacts indexFacts) - throws UnresolvedSearchChainException { - + IndexFacts indexFacts) throws UnresolvedSearchChainException { try { - return new LinkedHashSet<>(Arrays.asList(searchChainResolver.resolve(sourceRef, sourceToProviderMap))); + return new LinkedHashSet<>(List.of(searchChainResolver.resolve(sourceRef, sourceToProviderMap))); } catch (UnresolvedSourceRefException e) { return resolveClustersWithDocument(sourceRef, sourceToProviderMap, indexFacts); } @@ -65,9 +62,9 @@ public class SourceRefResolver { return searchChainResolver.resolve(new ComponentSpecification(cluster), sourceToProviderMap); } catch (UnresolvedSearchChainException e) { - throw new UnresolvedSearchChainException("Failed to resolve cluster search chain " + quote(cluster) + - " when using source ref " + quote(sourceRef) + - " as a document name."); + throw new UnresolvedSearchChainException("Failed to resolve cluster search chain '" + cluster + + "' when using source ref '" + sourceRef + + "' as a document name."); } } diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java index f23e24525bb..1b11e588f11 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java @@ -6,8 +6,6 @@ import com.yahoo.component.ComponentId; import com.yahoo.processing.request.Properties; /** - * TODO: What's this? - * * @author Tony Vaagenes */ public abstract class Target extends AbstractComponent { diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java index 5075b05454b..2ae923714c7 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java @@ -4,8 +4,6 @@ package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import static com.yahoo.container.util.Util.quote; - /** * @author Tony Vaagenes */ @@ -17,6 +15,6 @@ class UnresolvedProviderException extends UnresolvedSearchChainException { static UnresolvedSearchChainException createForMissingProvider(ComponentId source, ComponentSpecification provider) { - return new UnresolvedProviderException("No provider " + quote(provider) + " for source " + quote(source) + "."); + return new UnresolvedProviderException("No provider '" + provider + "' for source '" + source + "'."); } } diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java index 233e92c1699..6f0549eaf93 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java @@ -3,8 +3,6 @@ package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentSpecification; -import static com.yahoo.container.util.Util.quote; - /** * @author Tony Vaagenes */ @@ -16,6 +14,6 @@ class UnresolvedSourceRefException extends UnresolvedSearchChainException { static UnresolvedSearchChainException createForMissingSourceRef(ComponentSpecification source) { - return new UnresolvedSourceRefException("Could not resolve source ref " + quote(source) + "."); + return new UnresolvedSourceRefException("Could not resolve source ref '" + source + "'."); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java b/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java index d7ee3fedfc9..b74101fb83d 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java @@ -8,7 +8,7 @@ import com.yahoo.search.grouping.vespa.ContinuationDecoder; * subsequently be sent back along with the original request to navigate across a large result set. It is an opaque * data object that is not intended to be human readable.</p> * - * <p>To render a Cookie within a result set, you simply need to call {@link #toString()}.</p> + * <p>To render a continuation within a result set, you simply need to call {@link #toString()}.</p> * * @author Simon Thoresen Hult */ @@ -18,8 +18,8 @@ public abstract class Continuation { public static final String PREV_PAGE = "prev"; public static final String THIS_PAGE = "this"; - public static Continuation fromString(String str) { - return ContinuationDecoder.decode(str); + public static Continuation fromString(String string) { + return ContinuationDecoder.decode(string); } /** Returns a deep copy of this */ diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java index 9924a05bb46..b9e0825ab03 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java @@ -12,8 +12,14 @@ import com.yahoo.search.grouping.request.GroupingOperation; import com.yahoo.search.query.Select; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; +import com.yahoo.processing.IllegalInputException; -import java.util.*; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; /** * This searcher is responsible for turning the "select" parameter into a corresponding {@link GroupingRequest}. It will @@ -35,19 +41,23 @@ public class GroupingQueryParser extends Searcher { @Override public Result search(Query query, Execution execution) { - String reqParam = query.properties().getString(PARAM_REQUEST); - if (reqParam == null) { + try { + String reqParam = query.properties().getString(PARAM_REQUEST); + if (reqParam == null) return execution.search(query); + + List<Continuation> continuations = getContinuations(query.properties().getString(PARAM_CONTINUE)); + TimeZone zone = getTimeZone(query.properties().getString(PARAM_TIMEZONE, "utc")); + for (GroupingOperation op : GroupingOperation.fromStringAsList(reqParam)) { + GroupingRequest grpRequest = GroupingRequest.newInstance(query); + grpRequest.setRootOperation(op); + grpRequest.setTimeZone(zone); + grpRequest.continuations().addAll(continuations); + } return execution.search(query); } - List<Continuation> continuations = getContinuations(query.properties().getString(PARAM_CONTINUE)); - TimeZone zone = getTimeZone(query.properties().getString(PARAM_TIMEZONE, "utc")); - for (GroupingOperation op : GroupingOperation.fromStringAsList(reqParam)) { - GroupingRequest grpRequest = GroupingRequest.newInstance(query); - grpRequest.setRootOperation(op); - grpRequest.setTimeZone(zone); - grpRequest.continuations().addAll(continuations); + catch (IllegalArgumentException e) { + throw new IllegalInputException(e); } - return execution.search(query); } private List<Continuation> getContinuations(String param) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java index 13c23234910..8e57434a049 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java @@ -2,7 +2,6 @@ package com.yahoo.search.grouping; import com.yahoo.net.URI; -import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.grouping.request.GroupingOperation; @@ -12,7 +11,9 @@ import com.yahoo.search.grouping.result.RootId; import com.yahoo.search.query.Select; import com.yahoo.search.result.Hit; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.TimeZone; /** * An instance of this class represents one of many grouping requests that are attached to a {@link Query}. Use the diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java index 06b030dbc78..cd8578cd728 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java @@ -5,6 +5,7 @@ import com.google.inject.Inject; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Provides; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.grouping.request.AttributeMapLookupValue; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.container.QrSearchersConfig; @@ -80,14 +81,14 @@ public class GroupingValidator extends Searcher { AttributesConfig.Attribute keyAttribute = attributes.get(keyAttributeName); AttributesConfig.Attribute keySourceAttribute = attributes.get(keySourceAttributeName); if (!keySourceAttribute.datatype().equals(keyAttribute.datatype())) { - throw new IllegalArgumentException("Grouping request references key source attribute '" + - keySourceAttributeName + "' with data type '" + keySourceAttribute.datatype() + - "' that is different than data type '" + keyAttribute.datatype() + "' of key attribute '" + - keyAttributeName + "'"); + throw new IllegalInputException("Grouping request references key source attribute '" + + keySourceAttributeName + "' with data type '" + keySourceAttribute.datatype() + + "' that is different than data type '" + keyAttribute.datatype() + "' of key attribute '" + + keyAttributeName + "'"); } if (!keySourceAttribute.collectiontype().equals(AttributesConfig.Attribute.Collectiontype.Enum.SINGLE)) { - throw new IllegalArgumentException("Grouping request references key source attribute '" + - keySourceAttributeName + "' which is not of single value type"); + throw new IllegalInputException("Grouping request references key source attribute '" + + keySourceAttributeName + "' which is not of single value type"); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java index 8280917ca2f..3abfa3a6531 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java @@ -3,7 +3,7 @@ package com.yahoo.search.grouping; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -110,7 +110,7 @@ public class UniqueGroupingSearcher extends Searcher { if (null == root) { String msg = "Result group not found for deduping grouping request, returning empty result."; query.trace(msg, 3); - log.log(LogLevel.WARNING, msg); + log.log(Level.WARNING, msg); throw new IllegalStateException("Failed to produce deduped result set."); } result.hits().remove(root.getId().toString()); // hide our tracks diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java index 420861d2f6c..60805aacd5f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java @@ -38,9 +38,9 @@ public class AddFunction extends FunctionNode { /** * Constructs a new instance of this class from a list of arguments. * - * @param args The arguments to pass to the constructor. - * @return The created instance. - * @throws IllegalArgumentException Thrown if the number of arguments is less than 2. + * @param args the arguments to pass to the constructor. + * @return the created instance. + * @throws IllegalArgumentException thrown if the number of arguments is less than 2. */ public static AddFunction newInstance(List<GroupingExpression> args) { if (args.size() < 2) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/BucketResolver.java b/container-search/src/main/java/com/yahoo/search/grouping/request/BucketResolver.java index c36c8af5c34..6c6b20973ab 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/BucketResolver.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/BucketResolver.java @@ -24,9 +24,9 @@ public class BucketResolver { * Pushes the given expression onto this bucket resolver. Once all buckets have been pushed using this method, call * {@link #resolve(GroupingExpression)} to retrieve to combined grouping expression. * - * @param val The expression to push. - * @param inclusive Whether or not the value is inclusive or not. - * @throws IllegalArgumentException Thrown if the expression is incompatible. + * @param val the expression to push + * @param inclusive whether or not the value is inclusive or not + * @throws IllegalArgumentException thrown if the expression is incompatible */ public BucketResolver push(ConstantValue<?> val, boolean inclusive) { if (prev == null) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java index c825f3c61de..499ed610d34 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java @@ -8,7 +8,12 @@ import com.yahoo.search.grouping.request.parser.GroupingParserInput; import com.yahoo.search.grouping.request.parser.ParseException; import com.yahoo.search.grouping.request.parser.TokenMgrException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * This class represents a single node in a grouping operation tree. You may manually construct this tree, or you may @@ -228,8 +233,8 @@ public abstract class GroupingOperation extends GroupingNode { * method verifies the input level against the operation type, and recursively resolves the level of all argument * expressions. * - * @param level The level of the input data. - * @throws IllegalArgumentException Thrown if a contained expression is invalid for the given level. + * @param level the level of the input data + * @throws IllegalArgumentException thrown if a contained expression is invalid for the given level */ public void resolveLevel(int level) { if (groupBy != null) { @@ -322,12 +327,7 @@ public abstract class GroupingOperation extends GroupingNode { return this; } - /** - * Return the accuracy of this. - * - * @return The accuracy value. - * @see #setAccuracy(double) - */ + /** Return the accuracy of this. */ public double getAccuracy() { return accuracy; } @@ -335,8 +335,8 @@ public abstract class GroupingOperation extends GroupingNode { /** * Adds an expression to the order-by clause of this operation. * - * @param exp The expressions to add to this. - * @return This, to allow chaining. + * @param exp the expressions to add to this + * @return this, to allow chaining */ public GroupingOperation addOrderBy(GroupingExpression exp) { orderBy.add(exp); @@ -346,11 +346,11 @@ public abstract class GroupingOperation extends GroupingNode { /** * Convenience method to call {@link #addOrderBy(GroupingExpression)} for each element in the given list. * - * @param lst The list of expressions to add. - * @return This, to allow chaining. + * @param list the list of expressions to add + * @return this, to allow chaining */ - public GroupingOperation addOrderBy(List<GroupingExpression> lst) { - for (GroupingExpression exp : lst) { + public GroupingOperation addOrderBy(List<GroupingExpression> list) { + for (GroupingExpression exp : list) { addOrderBy(exp); } return this; @@ -359,7 +359,7 @@ public abstract class GroupingOperation extends GroupingNode { /** * Returns the number of expressions in the order-by clause of this. * - * @return The expression count. + * @return the expression count */ public int getNumOrderBy() { return orderBy.size(); @@ -368,9 +368,9 @@ public abstract class GroupingOperation extends GroupingNode { /** * Returns the group-by expression at the given index. * - * @param i The index of the expression to return. - * @return The expression at the given index. - * @throws IndexOutOfBoundsException If the index is out of range. + * @param i the index of the expression to return + * @return the expression at the given index + * @throws IndexOutOfBoundsException if the index is out of range */ public GroupingExpression getOrderBy(int i) { return orderBy.get(i); @@ -379,7 +379,7 @@ public abstract class GroupingOperation extends GroupingNode { /** * Returns an immutable view to the order-by clause of this. * - * @return The expression list. + * @return the expression list */ public List<GroupingExpression> getOrderBy() { return Collections.unmodifiableList(orderBy); @@ -388,8 +388,8 @@ public abstract class GroupingOperation extends GroupingNode { /** * Adds an expression to the output clause of this operation. * - * @param exp The expressions to add to this. - * @return This, to allow chaining. + * @param exp the expressions to add to this + * @return this, to allow chaining */ public GroupingOperation addOutput(GroupingExpression exp) { outputs.add(exp); @@ -399,8 +399,8 @@ public abstract class GroupingOperation extends GroupingNode { /** * Convenience method to call {@link #addOutput(GroupingExpression)} for each element in the given list. * - * @param lst The list of expressions to add. - * @return This, to allow chaining. + * @param lst the list of expressions to add + * @return this, to allow chaining */ public GroupingOperation addOutputs(List<GroupingExpression> lst) { for (GroupingExpression exp : lst) { @@ -412,7 +412,7 @@ public abstract class GroupingOperation extends GroupingNode { /** * Returns the number of expressions in the output clause of this. * - * @return The expression count. + * @return the expression count */ public int getNumOutputs() { return outputs.size(); @@ -421,9 +421,9 @@ public abstract class GroupingOperation extends GroupingNode { /** * Returns the output expression at the given index. * - * @param i The index of the expression to return. - * @return The expression at the given index. - * @throws IndexOutOfBoundsException If the index is out of range. + * @param i the index of the expression to return + * @return the expression at the given index + * @throws IndexOutOfBoundsException If the index is out of range */ public GroupingExpression getOutput(int i) { return outputs.get(i); @@ -432,7 +432,7 @@ public abstract class GroupingOperation extends GroupingNode { /** * Returns an immutable view to the output clause of this. * - * @return The expression list. + * @return the expression list */ public List<GroupingExpression> getOutputs() { return Collections.unmodifiableList(outputs); @@ -443,8 +443,8 @@ public abstract class GroupingOperation extends GroupingNode { * during expression evaluation to give the dispatch-node more data to consider when selecting the N groups that are * to be evaluated further. * - * @param precision The precision to set. - * @return This, to allow chaining. + * @param precision the precision to set + * @return this, to allow chaining * @see #setMax(int) */ public GroupingOperation setPrecision(int precision) { @@ -452,11 +452,7 @@ public abstract class GroupingOperation extends GroupingNode { return this; } - /** - * Returns the precision clause of this. - * - * @return The precision. - */ + /** Returns the precision clause of this. */ public int getPrecision() { return precision; } @@ -464,11 +460,11 @@ public abstract class GroupingOperation extends GroupingNode { /** * Assigns a string as the where clause of this operation. * - * @param str The string to assign to this. - * @return This, to allow chaining. + * @param string the string to assign to this + * @return this, to allow chaining */ - public GroupingOperation setWhere(String str) { - where = str; + public GroupingOperation setWhere(String string) { + where = string; return this; } @@ -590,9 +586,9 @@ public abstract class GroupingOperation extends GroupingNode { * Convenience method to call {@link #fromStringAsList(String)} and assert that the list contains exactly one * grouping operation. * - * @param str The string to parse. - * @return A grouping operation that corresponds to the string. - * @throws IllegalArgumentException Thrown if the string could not be parsed as a single operation. + * @param str the string to parse + * @return a grouping operation that corresponds to the string + * @throws IllegalArgumentException thrown if the string could not be parsed as a single operation */ public static GroupingOperation fromString(String str) { List<GroupingOperation> lst = fromStringAsList(str); @@ -606,15 +602,15 @@ public abstract class GroupingOperation extends GroupingNode { * Parses the given string as a list of grouping operations. This method never returns null, it either returns a * list of valid grouping requests or it throws an exception. * - * @param str The string to parse. - * @return A list of grouping operations that corresponds to the string. - * @throws IllegalArgumentException Thrown if the string could not be parsed. + * @param string the string to parse + * @return a list of grouping operations that corresponds to the string + * @throws IllegalArgumentException thrown if the string could not be parsed */ - public static List<GroupingOperation> fromStringAsList(String str) { - if (str == null || str.trim().length() == 0) { + public static List<GroupingOperation> fromStringAsList(String string) { + if (string == null || string.trim().length() == 0) { return Collections.emptyList(); } - GroupingParserInput input = new GroupingParserInput(str); + GroupingParserInput input = new GroupingParserInput(string); try { return new GroupingParser(input).requestList(); } catch (ParseException | TokenMgrException e) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MinFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MinFunction.java index 40fbda5a98a..5a39612b200 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MinFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MinFunction.java @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; /** - * This class represents a min-function in a {@link GroupingExpression}. It evaluates to a number that equals the + * A min-function in a {@link GroupingExpression}. It evaluates to a number that equals the * smallest of the results of all arguments. * * @author Simon Thoresen Hult @@ -16,9 +15,9 @@ public class MinFunction extends FunctionNode { /** * Constructs a new instance of this class. * - * @param arg1 The first compulsory argument, must evaluate to a number. - * @param arg2 The second compulsory argument, must evaluate to a number. - * @param argN The optional arguments, must evaluate to a number. + * @param arg1 the first compulsory argument, must evaluate to a number + * @param arg2 the second compulsory argument, must evaluate to a number + * @param argN the optional arguments, must evaluate to a number */ public MinFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { this(null, null, asList(arg1, arg2, argN)); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index 7c2e774f68b..d2d89e41879 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -13,7 +13,7 @@ import java.util.logging.Logger; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Provides; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.QueryCanonicalizer; @@ -225,9 +225,9 @@ public class GroupingExecutor extends Searcher { if (passList.isEmpty()) { throw new RuntimeException("No grouping request for pass " + pass + ", bug!"); } - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { for (Grouping grouping : passList) { - log.log(LogLevel.DEBUG, "Pass(" + pass + "), Grouping(" + grouping.getId() + "): " + grouping); + log.log(Level.FINE, "Pass(" + pass + "), Grouping(" + grouping.getId() + "): " + grouping); } } Item passRoot; @@ -263,9 +263,9 @@ public class GroupingExecutor extends Searcher { ret = passResult; } } - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { for (Grouping grouping : groupingMap.values()) { - log.log(LogLevel.DEBUG, "Result Grouping(" + grouping.getId() + "): " + grouping); + log.log(Level.FINE, "Result Grouping(" + grouping.getId() + "): " + grouping); } } return ret; diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java index e1a222d6bc0..15781060d7f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.vespa; +import java.util.Arrays; + /** * @author Simon Thoresen Hult */ @@ -33,7 +35,8 @@ class IntegerDecoder { if (c >= CHAR_MIN && c <= CHAR_MAX) { return (0xF & (c - CHAR_MIN)); } else { - throw new NumberFormatException(String.valueOf(c)); + throw new NumberFormatException("Expected a char in " + Arrays.toString(IntegerEncoder.CHARS) + + " but was '" + c + "'"); } } } diff --git a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java index d636d3bc925..ba034271a4c 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java +++ b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java @@ -1,20 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.handler; -import java.io.IOException; -import java.io.OutputStream; -import java.util.Collections; -import java.util.List; -import java.util.Map; - import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.collections.ListMap; -import com.yahoo.container.jdisc.ExtendedResponse; import com.yahoo.container.handler.Coverage; import com.yahoo.container.handler.Timing; +import com.yahoo.container.jdisc.ExtendedResponse; import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.container.logging.HitCounts; import com.yahoo.jdisc.HeaderFields; +import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.processing.execution.Execution.Trace.LogValue; @@ -25,6 +20,12 @@ import com.yahoo.search.Result; import com.yahoo.search.query.context.QueryContext; import com.yahoo.yolean.trace.TraceNode; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collections; +import java.util.List; +import java.util.Map; + /** * Wrap the result of a query as an HTTP response. * @@ -35,20 +36,21 @@ public class HttpSearchResponse extends ExtendedResponse { private final Result result; private final Query query; private final Renderer<Result> rendererCopy; + private final Metric metric; private final Timing timing; private final HitCounts hitCounts; private final TraceNode trace; - public HttpSearchResponse(int status, Result result, Query query, Renderer renderer) { - this(status, result, query, renderer, null); + public HttpSearchResponse(int status, Result result, Query query, Renderer<Result> renderer) { + this(status, result, query, renderer, null, null); } - HttpSearchResponse(int status, Result result, Query query, Renderer renderer, TraceNode trace) { + HttpSearchResponse(int status, Result result, Query query, Renderer<Result> renderer, TraceNode trace, Metric metric) { super(status); this.query = query; this.result = result; this.rendererCopy = renderer; - + this.metric = metric; this.timing = SearchResponse.createTiming(query, result); this.hitCounts = SearchResponse.createHitCounts(query, result); this.trace = trace; @@ -98,7 +100,11 @@ public class HttpSearchResponse extends ExtendedResponse { } try { try { - waitableRender(output); + long nanoStart = System.nanoTime(); + ListenableFuture<Boolean> promise = waitableRender(output); + if (metric != null) { + promise.addListener(new RendererLatencyReporter(nanoStart), Runnable::run); + } } finally { if (!(rendererCopy instanceof AsynchronousSectionedRenderer)) { output.flush(); @@ -173,9 +179,23 @@ public class HttpSearchResponse extends ExtendedResponse { @Override public Iterable<LogValue> getLogValues() { QueryContext context = query.getContext(false); - return context == null - ? Collections::emptyIterator - : context::logValueIterator; + return context == null ? Collections::emptyIterator : context::logValueIterator; + } + + private class RendererLatencyReporter implements Runnable { + + final long nanoStart; + + RendererLatencyReporter(long nanoStart) { this.nanoStart = nanoStart; } + + @Override + public void run() { + long latencyNanos = System.nanoTime() - nanoStart; + Metric.Context ctx = metric.createContext(Map.of( + SearchHandler.RENDERER_DIMENSION, rendererCopy.getClassName(), + SearchHandler.MIME_DIMENSION, rendererCopy.getMimeType())); + metric.set(SearchHandler.RENDER_LATENCY_METRIC, latencyNanos, ctx); + } } } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index dad106570ab..c15aef44f3d 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -10,45 +10,52 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.ContainerHttpConfig; +import com.yahoo.container.handler.threadpool.ContainerThreadPool; +import com.yahoo.container.jdisc.AclMapping; +import com.yahoo.container.jdisc.HttpMethodAclMapping; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.RequestHandlerSpec; import com.yahoo.container.jdisc.VespaHeaders; import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.Request; import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; +import com.yahoo.language.process.Embedder; import com.yahoo.net.HostName; import com.yahoo.net.UriTools; -import com.yahoo.prelude.query.QueryException; import com.yahoo.prelude.query.parser.ParseException; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.rendering.Renderer; import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.query.ranking.SoftTimeout; -import com.yahoo.search.searchchain.ExecutionFactory; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.yolean.Exceptions; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.query.context.QueryContext; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.search.query.properties.DefaultProperties; +import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.ExecutionFactory; import com.yahoo.search.searchchain.SearchChainRegistry; import com.yahoo.search.statistics.ElapsedTime; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.SlimeUtils; import com.yahoo.statistics.Callback; import com.yahoo.statistics.Handle; import com.yahoo.statistics.Statistics; import com.yahoo.statistics.Value; import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import com.yahoo.yolean.Exceptions; +import com.yahoo.yolean.trace.TraceNode; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -69,6 +76,8 @@ import java.util.logging.Logger; */ public class SearchHandler extends LoggingRequestHandler { + private static final Logger log = Logger.getLogger(SearchHandler.class.getName()); + private final AtomicInteger requestsInFlight = new AtomicInteger(0); // max number of threads for the executor for this handler @@ -77,15 +86,15 @@ public class SearchHandler extends LoggingRequestHandler { private static final CompoundName DETAILED_TIMING_LOGGING = new CompoundName("trace.timingDetails"); private static final CompoundName FORCE_TIMESTAMPS = new CompoundName("trace.timestamps"); - /** Event name for number of connections to the search subsystem */ private static final String SEARCH_CONNECTIONS = "search_connections"; + static final String RENDER_LATENCY_METRIC = "jdisc.render.latency"; + static final String MIME_DIMENSION = "mime"; + static final String RENDERER_DIMENSION = "renderer"; private static final String JSON_CONTENT_TYPE = "application/json"; - private static Logger log = Logger.getLogger(SearchHandler.class.getName()); - - private Value searchConnections; + private final Value searchConnections; public static final String defaultSearchChainName = "default"; private static final String fallbackSearchChain = "vespa"; @@ -97,10 +106,15 @@ public class SearchHandler extends LoggingRequestHandler { private final String selfHostname = HostName.getLocalhost(); + private final Embedder embedder; + private final ExecutionFactory executionFactory; private final AtomicLong numRequestsLeftToTrace; + private final static RequestHandlerSpec REQUEST_HANDLER_SPEC = RequestHandlerSpec.builder() + .withAclMapping(SearchHandler.aclRequestMapper()).build(); + private final class MeanConnections implements Callback { @Override @@ -117,43 +131,101 @@ public class SearchHandler extends LoggingRequestHandler { @Inject public SearchHandler(Statistics statistics, Metric metric, + ContainerThreadPool threadpool, + CompiledQueryProfileRegistry queryProfileRegistry, + ContainerHttpConfig config, + Embedder embedder, + ExecutionFactory executionFactory) { + this(statistics, metric, threadpool.executor(), queryProfileRegistry, embedder, executionFactory, + config.numQueriesToTraceOnDebugAfterConstruction(), + config.hostResponseHeaderKey().equals("") ? Optional.empty() : Optional.of(config.hostResponseHeaderKey())); + } + + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 + public SearchHandler(Statistics statistics, + Metric metric, + ContainerThreadPool threadpool, + AccessLog ignored, + CompiledQueryProfileRegistry queryProfileRegistry, + ContainerHttpConfig config, + ExecutionFactory executionFactory) { + this(statistics, metric, threadpool.executor(), ignored, queryProfileRegistry, config, executionFactory); + } + + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 + public SearchHandler(Statistics statistics, + Metric metric, Executor executor, - AccessLog accessLog, + AccessLog ignored, + CompiledQueryProfileRegistry queryProfileRegistry, + ContainerHttpConfig containerHttpConfig, + ExecutionFactory executionFactory) { + this(statistics, + metric, + executor, + queryProfileRegistry, + Embedder.throwsOnUse, + executionFactory, + containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), + containerHttpConfig.hostResponseHeaderKey().equals("") ? + Optional.empty() : Optional.of(containerHttpConfig.hostResponseHeaderKey())); + } + + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 + public SearchHandler(Statistics statistics, + Metric metric, + Executor executor, + AccessLog ignored, QueryProfilesConfig queryProfileConfig, ContainerHttpConfig containerHttpConfig, ExecutionFactory executionFactory) { this(statistics, metric, executor, - accessLog, QueryProfileConfigurer.createFromConfig(queryProfileConfig).compile(), + Embedder.throwsOnUse, executionFactory, containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), containerHttpConfig.hostResponseHeaderKey().equals("") ? - Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey())); + Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey())); } + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 public SearchHandler(Statistics statistics, Metric metric, Executor executor, - AccessLog accessLog, + AccessLog ignored, CompiledQueryProfileRegistry queryProfileRegistry, ExecutionFactory executionFactory, Optional<String> hostResponseHeaderKey) { - this(statistics, metric, executor, accessLog, queryProfileRegistry, executionFactory, 0, hostResponseHeaderKey); + this(statistics, metric, executor, queryProfileRegistry, Embedder.throwsOnUse, + executionFactory, 0, hostResponseHeaderKey); } private SearchHandler(Statistics statistics, Metric metric, Executor executor, - AccessLog accessLog, CompiledQueryProfileRegistry queryProfileRegistry, + Embedder embedder, ExecutionFactory executionFactory, long numQueriesToTraceOnDebugAfterStartup, Optional<String> hostResponseHeaderKey) { - super(executor, accessLog, metric, true); - log.log(LogLevel.DEBUG, "SearchHandler.init " + System.identityHashCode(this)); + super(executor, metric, true); + log.log(Level.FINE, () -> "SearchHandler.init " + System.identityHashCode(this)); this.queryProfileRegistry = queryProfileRegistry; + this.embedder = embedder; this.executionFactory = executionFactory; this.maxThreads = examineExecutor(executor); @@ -192,6 +264,8 @@ public class SearchHandler extends LoggingRequestHandler { new ExecutionFactory(chainsConfig, indexInfo, clusters, searchers, specialtokens, linguistics, renderers)); } + Metric metric() { return metric; } + private static int examineExecutor(Executor executor) { if (executor instanceof ThreadPoolExecutor) { return ((ThreadPoolExecutor) executor).getMaximumPoolSize(); @@ -205,10 +279,8 @@ public class SearchHandler extends LoggingRequestHandler { try { try { return handleBody(request); - } catch (QueryException e) { - return (e.getCause() instanceof IllegalArgumentException) - ? invalidParameterResponse(request, e) - : illegalQueryResponse(request, e); + } catch (IllegalInputException e) { + return illegalQueryResponse(request, e); } catch (RuntimeException e) { // Make sure we generate a valid response even on unexpected errors log.log(Level.WARNING, "Failed handling " + request, e); return internalServerErrorResponse(request, e); @@ -218,6 +290,9 @@ public class SearchHandler extends LoggingRequestHandler { } } + @Override + public Optional<Request.RequestType> getRequestType() { return Optional.of(Request.RequestType.READ); } + private int getHttpResponseStatus(com.yahoo.container.jdisc.HttpRequest httpRequest, Result result) { boolean benchmarkOutput = VespaHeaders.benchmarkOutput(httpRequest); if (benchmarkOutput) { @@ -234,15 +309,11 @@ public class SearchHandler extends LoggingRequestHandler { private HttpResponse errorResponse(HttpRequest request, ErrorMessage errorMessage) { Query query = new Query(); Result result = new Result(query, errorMessage); - Renderer renderer = getRendererCopy(ComponentSpecification.fromString(request.getProperty("format"))); + Renderer<Result> renderer = getRendererCopy(ComponentSpecification.fromString(request.getProperty("format"))); return new HttpSearchResponse(getHttpResponseStatus(request, result), result, query, renderer); } - private HttpResponse invalidParameterResponse(HttpRequest request, RuntimeException e) { - return errorResponse(request, ErrorMessage.createInvalidQueryParameter(Exceptions.toMessageString(e))); - } - private HttpResponse illegalQueryResponse(HttpRequest request, RuntimeException e) { return errorResponse(request, ErrorMessage.createIllegalQuery(Exceptions.toMessageString(e))); } @@ -251,7 +322,6 @@ public class SearchHandler extends LoggingRequestHandler { return errorResponse(request, ErrorMessage.createInternalServerError(Exceptions.toMessageString(e))); } - private HttpSearchResponse handleBody(HttpRequest request) { Map<String, String> requestMap = requestMapFromRequest(request); @@ -259,7 +329,11 @@ public class SearchHandler extends LoggingRequestHandler { String queryProfileName = requestMap.getOrDefault("queryProfile", null); CompiledQueryProfile queryProfile = queryProfileRegistry.findQueryProfile(queryProfileName); - Query query = new Query(request, requestMap, queryProfile); + Query query = new Query.Builder().setRequest(request) + .setRequestMap(requestMap) + .setQueryProfile(queryProfile) + .setEmbedder(embedder) + .build(); boolean benchmarking = VespaHeaders.benchmarkOutput(request); boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarking, request.getJDiscRequest().headers()); @@ -294,14 +368,13 @@ public class SearchHandler extends LoggingRequestHandler { } // Transform result to response - Renderer renderer = toRendererCopy(query.getPresentation().getRenderer()); + Renderer<Result> renderer = toRendererCopy(query.getPresentation().getRenderer()); HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result), result, query, renderer, - log.isLoggable(Level.FINE) - ? query.getContext(false).getTrace().traceNode() - : null); - if (hostResponseHeaderKey.isPresent()) - response.headers().add(hostResponseHeaderKey.get(), selfHostname); + extractTraceNode(query), + metric); + response.setRequestType(Request.RequestType.READ); + hostResponseHeaderKey.ifPresent(key -> response.headers().add(key, selfHostname)); if (benchmarking) VespaHeaders.benchmarkOutput(response.headers(), benchmarkCoverage, response.getTiming(), @@ -310,6 +383,19 @@ public class SearchHandler extends LoggingRequestHandler { return response; } + private static TraceNode extractTraceNode(Query query) { + if (log.isLoggable(Level.FINE)) { + QueryContext queryContext = query.getContext(false); + if (queryContext != null) { + Execution.Trace trace = queryContext.getTrace(); + if (trace != null) { + return trace.traceNode(); + } + } + } + return null; + } + private static int getErrors(Result result) { return result.hits().getErrorHit() == null ? 0 : 1; } @@ -404,7 +490,7 @@ public class SearchHandler extends LoggingRequestHandler { if (searchConnections != null) { connectionStatistics(); } else { - log.log(LogLevel.WARNING, + log.log(Level.WARNING, "searchConnections is a null reference, probably a known race condition during startup.", new IllegalStateException("searchConnections reference is null.")); } @@ -413,21 +499,17 @@ public class SearchHandler extends LoggingRequestHandler { } catch (ParseException e) { ErrorMessage error = ErrorMessage.createIllegalQuery("Could not parse query [" + request + "]: " + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, error::getDetailedMessage); + log.log(Level.FINE, error::getDetailedMessage); + return new Result(query, error); + } catch (IllegalInputException e) { + ErrorMessage error = ErrorMessage.createBadRequest("Invalid request [" + request + "]: " + + Exceptions.toMessageString(e)); + log.log(Level.FINE, error::getDetailedMessage); return new Result(query, error); } catch (IllegalArgumentException e) { - if ("Comparison method violates its general contract!".equals(e.getMessage())) { - // This is an error in application components or Vespa code - log(request, query, e); - return new Result(query, ErrorMessage.createUnspecifiedError("Failed searching: " + - Exceptions.toMessageString(e), e)); - } - else { - ErrorMessage error = ErrorMessage.createBadRequest("Invalid search request [" + request + "]: " - + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, error::getDetailedMessage); - return new Result(query, error); - } + log(request, query, e); + return new Result(query, ErrorMessage.createUnspecifiedError("Failed: " + + Exceptions.toMessageString(e), e)); } catch (LinkageError | StackOverflowError e) { // LinkageError should have been an Exception in an OSGi world - typical bundle dependency issue problem // StackOverflowError is recoverable @@ -437,7 +519,7 @@ public class SearchHandler extends LoggingRequestHandler { return new Result(query, error); } catch (Exception e) { log(request, query, e); - return new Result(query, ErrorMessage.createUnspecifiedError("Failed searching: " + + return new Result(query, ErrorMessage.createUnspecifiedError("Failed: " + Exceptions.toMessageString(e), e)); } } @@ -448,8 +530,8 @@ public class SearchHandler extends LoggingRequestHandler { if (maxThreads > 3) { // cast to long to avoid overflows if maxThreads is at no // log value (maxint) - final long maxThreadsAsLong = maxThreads; - final long connectionsAsLong = connections; + long maxThreadsAsLong = maxThreads; + long connectionsAsLong = connections; // only log when exactly crossing the limit to avoid // spamming the log if (connectionsAsLong < maxThreadsAsLong * 9L / 10L) { @@ -472,10 +554,10 @@ public class SearchHandler extends LoggingRequestHandler { private void log(String request, Query query, Throwable e) { // Attempted workaround for missing stack traces if (e.getStackTrace().length == 0) { - log.log(LogLevel.ERROR, "Failed executing " + query.toDetailString() + - " [" + request + "], received exception with no context", e); + log.log(Level.SEVERE, "Failed executing " + query.toDetailString() + + " [" + request + "], received exception with no context", e); } else { - log.log(LogLevel.ERROR, "Failed executing " + query.toDetailString() + " [" + request + "]", e); + log.log(Level.SEVERE, "Failed executing " + query.toDetailString() + " [" + request + "]", e); } } @@ -491,12 +573,13 @@ public class SearchHandler extends LoggingRequestHandler { if (query.getHits() > maxHits) { return new Result(query, ErrorMessage.createIllegalQuery(query.getHits() + - " hits requested, configured limit: " + maxHits + ".")); + " hits requested, configured limit: " + maxHits + + ". See https://docs.vespa.ai/en/reference/query-api-reference.html#native-execution-parameters")); } else if (query.getOffset() > maxOffset) { - return new Result(query, - ErrorMessage.createIllegalQuery("Offset of " + query.getOffset() + - " requested, configured limit: " + maxOffset + ".")); + return new Result(query, ErrorMessage.createIllegalQuery("Offset of " + query.getOffset() + + " requested, configured limit: " + maxOffset + + ". See https://docs.vespa.ai/en/reference/query-api-reference.html#native-execution-parameters")); } return null; } @@ -541,11 +624,12 @@ public class SearchHandler extends LoggingRequestHandler { Inspector inspector; try { - byte[] byteArray = IOUtils.readBytes(request.getData(), 1 << 20); + // Use an 4k buffer, that should be plenty for most json requests to pass in a single chunk + byte[] byteArray = IOUtils.readBytes(request.getData(), 4096); inspector = SlimeUtils.jsonToSlime(byteArray).get(); if (inspector.field("error_message").valid()) { - throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + - new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); + throw new IllegalInputException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); } } catch (IOException e) { @@ -558,9 +642,9 @@ public class SearchHandler extends LoggingRequestHandler { requestMap.putAll(request.propertyMap()); if (requestMap.containsKey("yql") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) - throw new QueryException("Illegal query: Query contains both yql and select parameter"); + throw new IllegalInputException("Illegal query: Query contains both yql and select parameter"); if (requestMap.containsKey("query") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) - throw new QueryException("Illegal query: Query contains both query and select parameter"); + throw new IllegalInputException("Illegal query: Query contains both query and select parameter"); return requestMap; } @@ -596,6 +680,17 @@ public class SearchHandler extends LoggingRequestHandler { }); } + @Override + public RequestHandlerSpec requestHandlerSpec() { + return REQUEST_HANDLER_SPEC; + } + + private static AclMapping aclRequestMapper() { + return HttpMethodAclMapping.standard() + .override(com.yahoo.jdisc.http.HttpRequest.Method.POST, AclMapping.Action.READ) + .build(); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchResponse.java b/container-search/src/main/java/com/yahoo/search/handler/SearchResponse.java index b4a469c569a..5b5ff0770c4 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchResponse.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchResponse.java @@ -49,9 +49,11 @@ public class SearchResponse { } public static Timing createTiming(Query query, Result result) { - return new Timing(result.getElapsedTime().firstFill(), + long summaryStartTime = result.getElapsedTime().firstFill(); + long queryStartTime = result.getElapsedTime().first(); + return new Timing(summaryStartTime, 0, - result.getElapsedTime().first(), + queryStartTime == Long.MAX_VALUE ? 0 : queryStartTime, query.getTimeout()); } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java index 0ec04bf99de..2074fce19bd 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java @@ -2,9 +2,9 @@ package com.yahoo.search.pagetemplates; import com.google.inject.Inject; -import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -13,8 +13,6 @@ import com.yahoo.search.pagetemplates.config.PageTemplateConfigurer; import com.yahoo.search.pagetemplates.engine.Organizer; import com.yahoo.search.pagetemplates.engine.Resolution; import com.yahoo.search.pagetemplates.engine.Resolver; -import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; -import com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver; import com.yahoo.search.pagetemplates.engine.resolvers.ResolverRegistry; import com.yahoo.search.pagetemplates.model.Choice; import com.yahoo.search.pagetemplates.model.PageElement; @@ -23,7 +21,13 @@ import com.yahoo.processing.request.CompoundName; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.Map; /** * Enables page optimization templates. @@ -107,7 +111,7 @@ public class PageTemplateSearcher extends Searcher { @Override public Result search(Query query, Execution execution) { // Pre execution: Choose template and sources - List<PageElement> pages=selectPageTemplates(query); + List<PageElement> pages = selectPageTemplates(query); if (pages.isEmpty()) return execution.search(query); // Bypass if no page template chosen addSources(pages,query); @@ -115,12 +119,12 @@ public class PageTemplateSearcher extends Searcher { query.properties().set(pagePageTemplateListName, pages); // Execute - Result result=execution.search(query); + Result result = execution.search(query); // Post execution: Resolve choices and organize the result as dictated by the resolved template - Choice pageTemplateChoice=Choice.createSingletons(pages); - Resolution resolution=selectResolver(query).resolve(pageTemplateChoice,query,result); - organizer.organize(pageTemplateChoice,resolution,result); + Choice pageTemplateChoice = Choice.createSingletons(pages); + Resolution resolution = selectResolver(query).resolve(pageTemplateChoice, query, result); + organizer.organize(pageTemplateChoice, resolution, result); return result; } @@ -132,23 +136,23 @@ public class PageTemplateSearcher extends Searcher { // Determine the list of page template ids @SuppressWarnings("unchecked") List<String> pageIds = (List<String>) query.properties().get(pageIdListName); - if (pageIds==null) { - String pageIdString=query.properties().getString(pageIdName,"").trim(); - if (pageIdString.length()>0) - pageIds=Arrays.asList(pageIdString.split(" ")); + if (pageIds == null) { + String pageIdString = query.properties().getString(pageIdName,"").trim(); + if (pageIdString.length() > 0) + pageIds = Arrays.asList(pageIdString.split(" ")); } // If none set, just return the default or null if none - if (pageIds==null) { + if (pageIds == null) { PageElement defaultPage=templateRegistry.getComponent("default"); - return (defaultPage==null ? Collections.<PageElement>emptyList() : Collections.singletonList(defaultPage)); + return (defaultPage == null ? Collections.<PageElement>emptyList() : Collections.singletonList(defaultPage)); } // Resolve the id list to page templates - List<PageElement> pages=new ArrayList<>(pageIds.size()); + List<PageElement> pages = new ArrayList<>(pageIds.size()); for (String pageId : pageIds) { - PageTemplate page=templateRegistry.getComponent(pageId); - if (page==null) + PageTemplate page = templateRegistry.getComponent(pageId); + if (page == null) query.errors().add(ErrorMessage.createInvalidQueryParameter("Could not resolve requested page template '" + pageId + "'")); else @@ -159,17 +163,17 @@ public class PageTemplateSearcher extends Searcher { } private Resolver selectResolver(Query query) { - String resolverId=query.properties().getString(pageResolverName); - if (resolverId==null) return resolverRegistry.defaultResolver(); - Resolver resolver=resolverRegistry.getComponent(resolverId); - if (resolver==null) throw new IllegalArgumentException("No page template resolver '" + resolverId + "'"); + String resolverId = query.properties().getString(pageResolverName); + if (resolverId == null) return resolverRegistry.defaultResolver(); + Resolver resolver = resolverRegistry.getComponent(resolverId); + if (resolver == null) throw new IllegalInputException("No page template resolver '" + resolverId + "'"); return resolver; } /** Sets query.getModel().getSources() to the right value and add source parameters specified in templates */ - private void addSources(List<PageElement> pages,Query query) { + private void addSources(List<PageElement> pages, Query query) { // Determine all wanted sources - Set<Source> pageSources=new HashSet<>(); + Set<Source> pageSources = new HashSet<>(); for (PageElement page : pages) pageSources.addAll(((PageTemplate)page).getSources()); @@ -177,34 +181,34 @@ public class PageTemplateSearcher extends Searcher { if (query.getModel().getSources().size() > 0) { // Add properties if the source list is set explicitly, but do not modify otherwise - addParametersForIncludedSources(pageSources,query); + addParametersForIncludedSources(pageSources, query); return; } if (pageSources.contains(Source.any)) { - IntentModel intentModel=IntentModel.getFrom(query); - if (intentModel!=null) { + IntentModel intentModel = IntentModel.getFrom(query); + if (intentModel != null) { query.getModel().getSources().addAll(intentModel.getSourceNames()); - addPageTemplateSources(pageSources,query); + addPageTemplateSources(pageSources, query); } // otherwise leave empty to search all } else { // Let the page templates decide - addPageTemplateSources(pageSources,query); + addPageTemplateSources(pageSources, query); } } private void addPageTemplateSources(Set<Source> pageSources,Query query) { for (Source pageSource : pageSources) { - if (pageSource==Source.any) continue; + if (pageSource == Source.any) continue; query.getModel().getSources().add(pageSource.getName()); addParameters(pageSource,query); } } - private void addParametersForIncludedSources(Set<Source> sources,Query query) { + private void addParametersForIncludedSources(Set<Source> sources, Query query) { for (Source source : sources) { - if (source.parameters().size()>0 && query.getModel().getSources().contains(source.getName())) + if (source.parameters().size() > 0 && query.getModel().getSources().contains(source.getName())) addParameters(source,query); } } @@ -220,8 +224,8 @@ public class PageTemplateSearcher extends Searcher { * is not supported. (Same parameter sets in multiple templates is supported, * and will be just one entry in this set). */ - private void addErrorIfSameSourceMultipleTimes(List<PageElement> pages,Set<Source> sources,Query query) { - Set<String> sourceNames=new HashSet<>(); + private void addErrorIfSameSourceMultipleTimes(List<PageElement> pages, Set<Source> sources, Query query) { + Set<String> sourceNames = new HashSet<>(); for (Source source : sources) { if (sourceNames.contains(source.getName())) query.errors().add(ErrorMessage.createInvalidQueryParameter( diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java index c29e9615fe8..c37ea5667c0 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java @@ -19,16 +19,16 @@ import java.util.Map; */ class PlaceholderMappingVisitor extends PageTemplateVisitor { - private Map<String, MapChoice> placeholderIdToChoice=new LinkedHashMap<>(); + private final Map<String, MapChoice> placeholderIdToChoice = new LinkedHashMap<>(); @Override public void visit(MapChoice mapChoice) { - List<String> placeholderIds=mapChoice.placeholderIds(); + List<String> placeholderIds = mapChoice.placeholderIds(); for (String placeholderId : placeholderIds) { - MapChoice existingChoice=placeholderIdToChoice.put(placeholderId,mapChoice); - if (existingChoice!=null) + MapChoice existingChoice = placeholderIdToChoice.put(placeholderId,mapChoice); + if (existingChoice != null) throw new IllegalArgumentException("placeholder id '" + placeholderId + "' is referenced by both " + - mapChoice + " and " + existingChoice + ": Only one reference is allowed"); + mapChoice + " and " + existingChoice + ": Only one reference is allowed"); } } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java index e8870f4f11a..5ef507201cb 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java @@ -1,9 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.pagetemplates; -import com.yahoo.search.pagetemplates.model.*; +import com.yahoo.search.pagetemplates.model.MapChoice; +import com.yahoo.search.pagetemplates.model.PageTemplateVisitor; +import com.yahoo.search.pagetemplates.model.Placeholder; -import java.util.HashMap; import java.util.Map; /** @@ -14,16 +15,16 @@ import java.util.Map; */ class PlaceholderReferenceCreatingVisitor extends PageTemplateVisitor { - private Map<String, MapChoice> placeholderIdToChoice=new HashMap<>(); + private final Map<String, MapChoice> placeholderIdToChoice; public PlaceholderReferenceCreatingVisitor(Map<String, MapChoice> placeholderIdToChoice) { - this.placeholderIdToChoice=placeholderIdToChoice; + this.placeholderIdToChoice = placeholderIdToChoice; } @Override public void visit(Placeholder placeholder) { - MapChoice choice=placeholderIdToChoice.get(placeholder.getId()); - if (choice==null) + MapChoice choice = placeholderIdToChoice.get(placeholder.getId()); + if (choice == null) throw new IllegalArgumentException(placeholder + " is not referenced by any choice"); placeholder.setValueContainer(choice); } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java index beffd12b22a..0359432a819 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java @@ -38,18 +38,18 @@ public class PageTemplateXMLReader { * @throws RuntimeException if <code>directory</code> is not a readable directory, or if there is some error in the XML */ public PageTemplateRegistry read(String directory) { - List<NamedReader> pageReaders=new ArrayList<>(); + List<NamedReader> pageReaders = new ArrayList<>(); try { - File dir=new File(directory); - if ( !dir.isDirectory() ) throw new IllegalArgumentException("Could not read page templates: '" + - directory + "' is not a valid directory."); + File dir = new File(directory); + if ( ! dir.isDirectory() ) throw new IllegalArgumentException("Could not read page templates: '" + + directory + "' is not a valid directory."); for (File file : sortFiles(dir)) { if ( ! file.getName().endsWith(".xml")) continue; - pageReaders.add(new NamedReader(file.getName(),new FileReader(file))); + pageReaders.add(new NamedReader(file.getName(), new FileReader(file))); } - return read(pageReaders,true); + return read(pageReaders, true); } catch (IOException e) { throw new IllegalArgumentException("Could not read page templates from '" + directory + "'",e); @@ -67,18 +67,18 @@ public class PageTemplateXMLReader { * @throws RuntimeException if <code>fileName</code> is not a readable file, or if there is some error in the XML */ public PageTemplate readFile(String fileName) { - NamedReader pageReader=null; + NamedReader pageReader = null; try { - File file=new File(fileName); - pageReader=new NamedReader(fileName,new FileReader(file)); - String firstName=file.getName().substring(0,file.getName().length()-4); - return read(Collections.singletonList(pageReader),true).getComponent(firstName); + File file = new File(fileName); + pageReader = new NamedReader(fileName,new FileReader(file)); + String firstName = file.getName().substring(0, file.getName().length() - 4); + return read(Collections.singletonList(pageReader), true).getComponent(firstName); } catch (IOException e) { - throw new IllegalArgumentException("Could not read the page template '" + fileName + "'",e); + throw new IllegalArgumentException("Could not read the page template '" + fileName + "'", e); } finally { - if (pageReader!=null) + if (pageReader != null) try { pageReader.close(); } catch (IOException e) { } } } @@ -130,11 +130,11 @@ public class PageTemplateXMLReader { } /** Throws an exception if the name is not corresponding to the id */ - private void validateFileName(final String actualName,ComponentId id,String artifactName) { - String expectedCanonicalFileName=id.toFileName(); - String fileName=new File(actualName).getName(); - fileName=stripXmlEnding(fileName); - String canonicalFileName=ComponentId.fromFileName(fileName).toFileName(); + private void validateFileName(String actualName, ComponentId id, String artifactName) { + String expectedCanonicalFileName = id.toFileName(); + String fileName = new File(actualName).getName(); + fileName = stripXmlEnding(fileName); + String canonicalFileName = ComponentId.fromFileName(fileName).toFileName(); if ( ! canonicalFileName.equals(expectedCanonicalFileName)) throw new IllegalArgumentException("The file name of " + artifactName + " '" + id + "' must be '" + expectedCanonicalFileName + ".xml' but was '" + actualName + "'"); @@ -144,14 +144,14 @@ public class PageTemplateXMLReader { if (!fileName.endsWith(".xml")) throw new IllegalArgumentException("'" + fileName + "' should have a .xml ending"); else - return fileName.substring(0,fileName.length()-4); + return fileName.substring(0, fileName.length() - 4); } private void readPages() { for (Map.Entry<ComponentId,Element> pageElement : pageElementsByPageId.entrySet()) { try { - PageTemplate page=registry.getComponent(pageElement.getValue().getAttribute("id")); - readPageContent(pageElement.getValue(),page); + PageTemplate page = registry.getComponent(pageElement.getValue().getAttribute("id")); + readPageContent(pageElement.getValue(), page); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Could not read page template '" + pageElement.getKey() + "'",e); @@ -159,16 +159,16 @@ public class PageTemplateXMLReader { } } - private void readPageContent(Element pageElement,PageTemplate page) { + private void readPageContent(Element pageElement, PageTemplate page) { if (page.isFrozen()) return; // Already read - Section rootSection=new Section(page.getId().toString()); - readSection(pageElement,rootSection); + Section rootSection = new Section(page.getId().toString()); + readSection(pageElement, rootSection); page.setSection(rootSection); page.freeze(); } /** Fills a section with attributes and sub-elements from a "section" or "page" element */ - private Section readSection(Element sectionElement,Section section) { + private Section readSection(Element sectionElement, Section section) { section.setLayout(Layout.fromString(sectionElement.getAttribute("layout"))); section.setRegion(sectionElement.getAttribute("region")); section.setOrder(Sorting.fromString(sectionElement.getAttribute("order"))); @@ -198,10 +198,10 @@ public class PageTemplateXMLReader { /** Reads the direct descendant elements of an include */ private List<PageElement> readInclude(Element element) { - PageTemplate included=registry.getComponent(element.getAttribute("idref")); - if (included==null) + PageTemplate included = registry.getComponent(element.getAttribute("idref")); + if (included == null) throw new IllegalArgumentException("Could not find page template '" + element.getAttribute("idref")); - readPageContent(pageElementsByPageId.get(included.getId()),included); + readPageContent(pageElementsByPageId.get(included.getId()), included); return included.getSection().elements(Section.class); } @@ -223,9 +223,9 @@ public class PageTemplateXMLReader { } private List<Source> readSourceAttribute(Element sectionElement) { - List<Source> sources=new ArrayList<>(); - String sourceAttributeString=sectionElement.getAttribute("source"); - if (sourceAttributeString!=null) { + List<Source> sources = new ArrayList<>(); + String sourceAttributeString = sectionElement.getAttribute("source"); + if (sourceAttributeString != null) { for (String sourceName : sourceAttributeString.split(" ")) { if (sourceName.isEmpty()) continue; if ("*".equals(sourceName)) diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Organizer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Organizer.java index 3e6e82a5584..051103cba06 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Organizer.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Organizer.java @@ -9,9 +9,7 @@ import com.yahoo.search.query.Sorting; import com.yahoo.search.result.*; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import java.util.Map; /** * Reorganizes and prunes a result as prescribed by a resolved template. @@ -29,13 +27,13 @@ public class Organizer { * @param result the result to organize */ public void organize(Choice templateChoice, Resolution resolution, Result result) { - PageTemplate template=(PageTemplate)templateChoice.get(resolution.getResolution(templateChoice)).get(0); - SectionHitGroup sectionGroup =toGroup(template.getSection(),resolution,result); - ErrorHit errors=result.hits().getErrorHit(); + PageTemplate template = (PageTemplate)templateChoice.get(resolution.getResolution(templateChoice)).get(0); + SectionHitGroup sectionGroup = toGroup(template.getSection(), resolution, result); + ErrorHit errors = result.hits().getErrorHit(); // transfer state from existing hit sectionGroup.setQuery(result.hits().getQuery()); - if (errors!=null && errors instanceof DefaultErrorHit) + if (errors instanceof DefaultErrorHit) sectionGroup.add((DefaultErrorHit)errors); result.hits().forEachField((name, value) -> sectionGroup.setField(name, value)); result.setHits(sectionGroup); diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java index e0a3821e10c..f36ebe56e21 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java @@ -40,10 +40,10 @@ public class Resolution { * been resolved in this */ public int getResolution(Choice choice) { - if (choice.alternatives().size()==1) return 0; + if (choice.alternatives().size() == 1) return 0; if (choice.isEmpty()) throw new IllegalArgumentException("Cannot return a resolution of empty " + choice); - Integer resolution=choiceResolutions.get(choice); - if (resolution==null) throw new IllegalArgumentException(this + " has no resolution of " + choice); + Integer resolution = choiceResolutions.get(choice); + if (resolution == null) throw new IllegalArgumentException(this + " has no resolution of " + choice); return resolution; } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java index f2e2e1b034d..47914792da8 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java @@ -271,10 +271,10 @@ public class PageTemplatesXmlRenderer extends AsynchronousSectionedRenderer<Resu private Result getResult() { try { - return (Result) getResponse(); + return (Result)getResponse(); } catch (ClassCastException e) { - throw new IllegalArgumentException("PageTemplatesXmlRenderer attempted used outside a search context, got a " + - getResponse().getClass().getName()); + throw new IllegalStateException("PageTemplatesXmlRenderer attempted used outside a search context, got a " + + getResponse().getClass().getName()); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index f06aab09a3d..637873aa375 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -7,6 +7,7 @@ import com.yahoo.language.LocaleFactory; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.TaggableItem; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.query.parser.Parsable; @@ -236,10 +237,14 @@ public class Model implements Cloneable { */ public QueryTree getQueryTree() { if (queryTree == null) { - Parser parser = ParserFactory.newInstance(type, ParserEnvironment.fromExecutionContext(execution.context())); - queryTree = parser.parse(Parsable.fromQueryModel(this)); - if (parent.getTraceLevel() >= 2) { - parent.trace("Query parsed to: " + parent.yqlRepresentation(), 2); + try { + Parser parser = ParserFactory.newInstance(type, ParserEnvironment.fromExecutionContext(execution.context())); + queryTree = parser.parse(Parsable.fromQueryModel(this)); + if (parent.getTraceLevel() >= 2) + parent.trace("Query parsed to: " + parent.yqlRepresentation(), 2); + } + catch (IllegalArgumentException e) { + throw new IllegalInputException("Failed parsing query", e); } } return queryTree; diff --git a/container-search/src/main/java/com/yahoo/search/query/ParameterParser.java b/container-search/src/main/java/com/yahoo/search/query/ParameterParser.java index d358fa06977..c0f0ee80730 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ParameterParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/ParameterParser.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query; -import static com.yahoo.container.util.Util.quote; +import com.yahoo.processing.IllegalInputException; /** * Wrapper class to avoid code duplication of common parsing requirements. @@ -25,13 +25,8 @@ public class ParameterParser { * representation cannot be parsed as a number followed optionally by time unit */ public static Long asMilliSeconds(Object value, Long defaultValue) { - if (value == null) { - return defaultValue; - } - if (value instanceof Number) { - Number n = (Number) value; - return Long.valueOf(n.longValue() * 1000L); - } + if (value == null) return defaultValue; + if (value instanceof Number) return ((Number)value).longValue() * 1000L; return parseTime(value.toString()); } @@ -43,7 +38,7 @@ public class ParameterParser { double multiplier = parseUnit(time.substring(unitOffset)); return (long) (measure * multiplier); } catch (RuntimeException e) { - throw new IllegalArgumentException("Error parsing " + quote(time), e); + throw new IllegalInputException("Error parsing '" + time + "'", e); } } @@ -58,7 +53,7 @@ public class ParameterParser { } } if (unitOffset == 0) { - throw new NumberFormatException("Invalid number " + quote(time)); + throw new IllegalInputException("Invalid number '" + time + "'"); } return unitOffset; } diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index 13196488e98..db2fbf525e0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -23,7 +23,7 @@ import java.util.Set; public class Presentation implements Cloneable { /** The type representing the property arguments consumed by this */ - private static QueryProfileType argumentType; + private static final QueryProfileType argumentType; public static final String PRESENTATION = "presentation"; public static final String BOLDING = "bolding"; @@ -48,7 +48,7 @@ public class Presentation implements Cloneable { public static QueryProfileType getArgumentType() { return argumentType; } /** How the result should be highlighted */ - private Highlight highlight= null; + private Highlight highlight = null; /** The terms to highlight in the result (only used by BoldingSearcher, may be removed later). */ private List<IndexedItem> boldingData = null; @@ -128,15 +128,15 @@ public class Presentation implements Cloneable { return clone; } catch (CloneNotSupportedException e) { - throw new RuntimeException("Someone inserted a noncloneable superclass",e); + throw new RuntimeException("Someone inserted a noncloneable superclass", e); } } @Override public boolean equals(Object o) { - if (o == null || !(o instanceof Presentation)) return false; + if ( ! (o instanceof Presentation)) return false; Presentation p = (Presentation) o; - return QueryHelper.equals(bolding,p.bolding) && QueryHelper.equals(summary,p.summary); + return QueryHelper.equals(bolding, p.bolding) && QueryHelper.equals(summary, p.summary); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/query/Properties.java b/container-search/src/main/java/com/yahoo/search/query/Properties.java index a1a70b4c3ba..a0cd4137e9f 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Properties.java +++ b/container-search/src/main/java/com/yahoo/search/query/Properties.java @@ -30,8 +30,9 @@ public abstract class Properties extends com.yahoo.processing.request.Properties return (Properties)super.clone(); } - /** The query owning this property object. - * Only guaranteed to work if this instance is accessible as query.properties() + /** + * Returns the query owning this property object. + * Only guaranteed to work if this instance is accessible as query.properties() */ public Query getParentQuery() { if (chained() == null) { @@ -48,4 +49,5 @@ public abstract class Properties extends com.yahoo.processing.request.Properties if (chained() != null) chained().setParentQuery(query); } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index 830a3f4ef81..0aa3f12ca68 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -39,6 +39,7 @@ public class Ranking implements Cloneable { public static final String LIST_FEATURES = "listFeatures"; public static final String FRESHNESS = "freshness"; public static final String QUERYCACHE = "queryCache"; + public static final String RERANKCOUNT = "rerankCount"; public static final String MATCH_PHASE = "matchPhase"; public static final String DIVERSITY = "diversity"; public static final String SOFTTIMEOUT = "softtimeout"; @@ -55,7 +56,8 @@ public class Ranking implements Cloneable { argumentType.addField(new FieldDescription(SORTING, "string", "sorting sortspec")); argumentType.addField(new FieldDescription(LIST_FEATURES, "string", RANKFEATURES.toString())); argumentType.addField(new FieldDescription(FRESHNESS, "string", "datetime")); - argumentType.addField(new FieldDescription(QUERYCACHE, "string")); + argumentType.addField(new FieldDescription(QUERYCACHE, "boolean")); + argumentType.addField(new FieldDescription(RERANKCOUNT, "integer")); argumentType.addField(new FieldDescription(MATCH_PHASE, new QueryProfileFieldType(MatchPhase.getArgumentType()), "matchPhase")); argumentType.addField(new FieldDescription(DIVERSITY, new QueryProfileFieldType(Diversity.getArgumentType()))); argumentType.addField(new FieldDescription(SOFTTIMEOUT, new QueryProfileFieldType(SoftTimeout.getArgumentType()))); @@ -67,7 +69,7 @@ public class Ranking implements Cloneable { } public static QueryProfileType getArgumentType() { return argumentType; } - private Query parent; + private final Query parent; /** The location of the query is used for distance ranking */ private Location location = null; @@ -85,6 +87,8 @@ public class Ranking implements Cloneable { private boolean queryCache = false; + private Integer rerankCount = null; + private RankProperties rankProperties = new RankProperties(); private RankFeatures rankFeatures = new RankFeatures(); @@ -140,6 +144,15 @@ public class Ranking implements Cloneable { public boolean getQueryCache() { return queryCache; } + /** + * Sets the number of hits for which the second-phase function will be evaluated. + * When set, this overrides the setting in the rank profile. + */ + public void setRerankCount(int rerankCount) { this.rerankCount = rerankCount; } + + /** Returns the rerank-count that will be used, or null if not set */ + public Integer getRerankCount() { return rerankCount; } + /** Returns the location of this query, or null if none */ public Location getLocation() { return location; } @@ -258,6 +271,8 @@ public class Ranking implements Cloneable { matching.prepare(rankProperties); softTimeout.prepare(rankProperties); prepareNow(freshness); + if (rerankCount != null) + rankProperties.put("vespa.hitcollector.heapsize", rerankCount); } private void prepareNow(Freshness freshness) { diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java index cb662dcd671..a7e491f5269 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Select.java +++ b/container-search/src/main/java/com/yahoo/search/query/Select.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query; -import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.query.parser.ParserEnvironment; @@ -12,13 +11,12 @@ import com.yahoo.search.yql.VespaGroupingStep; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; /** - * The parameters defining the where-clause and groping of a query + * The parameters defining the where-clause and grouping of a query * * @author henrhoi */ @@ -26,7 +24,6 @@ public class Select implements Cloneable { /** The type representing the property arguments consumed by this */ private static final QueryProfileType argumentType; - private static final CompoundName argumentTypeName; public static final String SELECT = "select"; public static final String WHERE = "where"; @@ -46,7 +43,6 @@ public class Select implements Cloneable { argumentType.addField(new FieldDescription(WHERE, "string")); argumentType.addField(new FieldDescription(GROUPING, "string")); argumentType.freeze(); - argumentTypeName = new CompoundName(argumentType.getId().getName()); } public static QueryProfileType getArgumentType() { return argumentType; } @@ -57,12 +53,13 @@ public class Select implements Cloneable { } public Select(String where, String grouping, Query query) { - this(where, grouping, query, Collections.emptyList()); + this(where, grouping, null, query, Collections.emptyList()); } - private Select(String where, String grouping, Query query, List<GroupingRequest> groupingRequests) { + private Select(String where, String grouping, String groupingExpressionString, Query query, List<GroupingRequest> groupingRequests) { this.where = Objects.requireNonNull(where, "A Select must have a where string (possibly the empty string)"); this.grouping = Objects.requireNonNull(grouping, "A Select must have a select string (possibly the empty string)"); + this.groupingExpressionString = groupingExpressionString; this.parent = Objects.requireNonNull(query, "A Select must have a parent query"); this.groupingRequests = deepCopy(groupingRequests, this); } @@ -78,7 +75,7 @@ public class Select implements Cloneable { * Sets the document selection criterion of the query. * * @param where the documents to select as a JSON string on the format specified in - * <a href="https://docs.vespa.ai/documentation/reference/select-reference.html">the select reference doc</a> + * <a href="https://docs.vespa.ai/en/reference/select-reference.html">the select reference doc</a> */ public void setWhereString(String where) { this.where = where; @@ -95,7 +92,7 @@ public class Select implements Cloneable { * Sets the grouping operation of the query. * * @param grouping the grouping to perform as a JSON string on the format specified in - * <a href="https://docs.vespa.ai/documentation/reference/select-reference.html">the select reference doc</a> + * <a href="https://docs.vespa.ai/en/reference/select-reference.html">the select reference doc</a> */ public void setGroupingString(String grouping) { groupingRequests.clear(); @@ -131,16 +128,16 @@ public class Select implements Cloneable { @Override public String toString() { - return "where: [" + where + "], grouping: [" + grouping+ "]"; + return "where: [" + where + "], grouping: [" + grouping + "]"; } @Override public Object clone() { - return new Select(where, grouping, parent, groupingRequests); + return new Select(where, grouping, groupingExpressionString, parent, groupingRequests); } public Select cloneFor(Query parent) { - return new Select(where, grouping, parent, groupingRequests); + return new Select(where, grouping, groupingExpressionString, parent, groupingRequests); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 775dca7c444..f94c6300e83 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -2,10 +2,14 @@ package com.yahoo.search.query; import com.google.common.base.Preconditions; +import com.yahoo.processing.IllegalInputException; import com.yahoo.collections.LazyMap; +import com.yahoo.geo.DistanceParser; +import com.yahoo.geo.ParsedDegree; import com.yahoo.language.Language; import com.yahoo.language.process.Normalizer; import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.Location; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.BoolItem; import com.yahoo.prelude.query.CompositeItem; @@ -15,14 +19,15 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; import com.yahoo.prelude.query.ONearItem; import com.yahoo.prelude.query.OrItem; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.PredicateQueryItem; import com.yahoo.prelude.query.PrefixItem; -import com.yahoo.prelude.query.QueryException; import com.yahoo.prelude.query.RangeItem; import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.RegExpItem; @@ -46,6 +51,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.SlimeUtils; +import com.yahoo.slime.Type; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -60,6 +66,59 @@ import static com.yahoo.slime.Type.LONG; import static com.yahoo.slime.Type.OBJECT; import static com.yahoo.slime.Type.STRING; +import static com.yahoo.search.yql.YqlParser.ACCENT_DROP; +import static com.yahoo.search.yql.YqlParser.ALTERNATIVES; +import static com.yahoo.search.yql.YqlParser.AND_SEGMENTING; +import static com.yahoo.search.yql.YqlParser.ANNOTATIONS; +import static com.yahoo.search.yql.YqlParser.APPROXIMATE; +import static com.yahoo.search.yql.YqlParser.ASCENDING_HITS_ORDER; +import static com.yahoo.search.yql.YqlParser.CONNECTION_ID; +import static com.yahoo.search.yql.YqlParser.CONNECTION_WEIGHT; +import static com.yahoo.search.yql.YqlParser.CONNECTIVITY; +import static com.yahoo.search.yql.YqlParser.DEFAULT_TARGET_NUM_HITS; +import static com.yahoo.search.yql.YqlParser.DESCENDING_HITS_ORDER; +import static com.yahoo.search.yql.YqlParser.DISTANCE; +import static com.yahoo.search.yql.YqlParser.DISTANCE_THRESHOLD; +import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; +import static com.yahoo.search.yql.YqlParser.EQUIV; +import static com.yahoo.search.yql.YqlParser.FILTER; +import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; +import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; +import static com.yahoo.search.yql.YqlParser.HNSW_EXPLORE_ADDITIONAL_HITS; +import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; +import static com.yahoo.search.yql.YqlParser.LABEL; +import static com.yahoo.search.yql.YqlParser.NEAR; +import static com.yahoo.search.yql.YqlParser.NEAREST_NEIGHBOR; +import static com.yahoo.search.yql.YqlParser.NFKC; +import static com.yahoo.search.yql.YqlParser.NORMALIZE_CASE; +import static com.yahoo.search.yql.YqlParser.ONEAR; +import static com.yahoo.search.yql.YqlParser.ORIGIN; +import static com.yahoo.search.yql.YqlParser.ORIGIN_LENGTH; +import static com.yahoo.search.yql.YqlParser.ORIGIN_OFFSET; +import static com.yahoo.search.yql.YqlParser.ORIGIN_ORIGINAL; +import static com.yahoo.search.yql.YqlParser.PHRASE; +import static com.yahoo.search.yql.YqlParser.PREDICATE; +import static com.yahoo.search.yql.YqlParser.PREFIX; +import static com.yahoo.search.yql.YqlParser.RANGE; +import static com.yahoo.search.yql.YqlParser.RANK; +import static com.yahoo.search.yql.YqlParser.RANKED; +import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT; +import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD; +import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE; +import static com.yahoo.search.yql.YqlParser.STEM; +import static com.yahoo.search.yql.YqlParser.SUBSTRING; +import static com.yahoo.search.yql.YqlParser.SUFFIX; +import static com.yahoo.search.yql.YqlParser.TARGET_HITS; +import static com.yahoo.search.yql.YqlParser.TARGET_NUM_HITS; +import static com.yahoo.search.yql.YqlParser.THRESHOLD_BOOST_FACTOR; +import static com.yahoo.search.yql.YqlParser.UNIQUE_ID; +import static com.yahoo.search.yql.YqlParser.USE_POSITION_DATA; +import static com.yahoo.search.yql.YqlParser.USER_INPUT_LANGUAGE; +import static com.yahoo.search.yql.YqlParser.WAND; +import static com.yahoo.search.yql.YqlParser.WEAK_AND; +import static com.yahoo.search.yql.YqlParser.WEIGHT; +import static com.yahoo.search.yql.YqlParser.WEIGHTED_SET; + /** * The Select query language. * @@ -69,6 +128,14 @@ import static com.yahoo.slime.Type.STRING; */ public class SelectParser implements Parser { + private static final String AND = "and"; + private static final String AND_NOT = "and_not"; + private static final String CALL = "call"; + private static final String CONTAINS = "contains"; + private static final String EQ = "equals"; + private static final String MATCHES = "matches"; + private static final String OR = "or"; + Parsable query; private final IndexFacts indexFacts; private final Map<Integer, TaggableItem> identifiedItems = LazyMap.newHashMap(); @@ -76,61 +143,7 @@ public class SelectParser implements Parser { private final Normalizer normalizer; private IndexFacts.Session indexFactsSession; - // YQL parameters and functions - private static final String DESCENDING_HITS_ORDER = "descending"; - private static final String ASCENDING_HITS_ORDER = "ascending"; - private static final Integer DEFAULT_TARGET_NUM_HITS = 10; - private static final String ORIGIN_LENGTH = "length"; - private static final String ORIGIN_OFFSET = "offset"; - private static final String ORIGIN = "origin"; - private static final String ORIGIN_ORIGINAL = "original"; - private static final String CONNECTION_ID = "id"; - private static final String CONNECTION_WEIGHT = "weight"; - private static final String CONNECTIVITY = "connectivity"; - private static final String ANNOTATIONS = "annotations"; - private static final String NFKC = "nfkc"; - private static final String USER_INPUT_LANGUAGE = "language"; - private static final String ACCENT_DROP = "accentDrop"; - private static final String ALTERNATIVES = "alternatives"; - private static final String AND_SEGMENTING = "andSegmenting"; - private static final String DISTANCE = "distance"; - private static final String DOT_PRODUCT = "dotProduct"; - private static final String EQUIV = "equiv"; - private static final String FILTER = "filter"; - private static final String HIT_LIMIT = "hitLimit"; - private static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; - private static final String LABEL = "label"; - private static final String NEAR = "near"; - private static final String NORMALIZE_CASE = "normalizeCase"; - private static final String ONEAR = "onear"; - private static final String PHRASE = "phrase"; - private static final String PREDICATE = "predicate"; - private static final String PREFIX = "prefix"; - private static final String RANKED = "ranked"; - private static final String RANK = "rank"; - private static final String SAME_ELEMENT = "sameElement"; - private static final String SCORE_THRESHOLD = "scoreThreshold"; - private static final String SIGNIFICANCE = "significance"; - private static final String STEM = "stem"; - private static final String SUBSTRING = "substring"; - private static final String SUFFIX = "suffix"; - private static final String TARGET_NUM_HITS = "targetNumHits"; - private static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; - private static final String UNIQUE_ID = "id"; - private static final String USE_POSITION_DATA = "usePositionData"; - private static final String WAND = "wand"; - private static final String WEAK_AND = "weakAnd"; - private static final String WEIGHTED_SET = "weightedSet"; - private static final String WEIGHT = "weight"; - private static final String AND = "and"; - private static final String AND_NOT = "and_not"; - private static final String OR = "or"; - private static final String EQ = "equals"; - private static final String RANGE = "range"; - private static final String CONTAINS = "contains"; - private static final String MATCHES = "matches"; - private static final String CALL = "call"; - private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, PREDICATE, RANK, WEAK_AND); + private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, GEO_LOCATION, NEAREST_NEIGHBOR, PREDICATE, RANK, WEAK_AND); public SelectParser(ParserEnvironment environment) { indexFacts = environment.getIndexFacts(); @@ -148,15 +161,20 @@ public class SelectParser implements Parser { } private QueryTree buildTree() { - Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString().getBytes()).get(); + Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString()).get(); if (inspector.field("error_message").valid()) { - throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + + throw new IllegalInputException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); } - Item root = walkJson(inspector); - connectItems(); - return new QueryTree(root); + try { + Item root = walkJson(inspector); + connectItems(); + return new QueryTree(root); + } + catch (IllegalArgumentException e) { + throw new IllegalInputException("Illegal JSON query", e); + } } private Item walkJson(Inspector inspector) { @@ -208,10 +226,10 @@ public class SelectParser implements Parser { /** Translates a list of grouping requests on JSON form to a list in the grouping language form */ private List<String> toGroupingRequests(String groupingJson) { - Inspector inspector = SlimeUtils.jsonToSlime(groupingJson.getBytes()).get(); + Inspector inspector = SlimeUtils.jsonToSlime(groupingJson).get(); if (inspector.field("error_message").valid()) { - throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + - " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); + throw new IllegalInputException("Illegal query: " + inspector.field("error_message").asString() + + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); } List<String> operations = new ArrayList<>(); @@ -259,6 +277,10 @@ public class SelectParser implements Parser { return buildWeightedSet(key, value); case DOT_PRODUCT: return buildDotProduct(key, value); + case GEO_LOCATION: + return buildGeoLocation(key, value); + case NEAREST_NEIGHBOR: + return buildNearestNeighbor(key, value); case PREDICATE: return buildPredicate(key, value); case RANK: @@ -266,7 +288,7 @@ public class SelectParser implements Parser { case WEAK_AND: return buildWeakAnd(key, value); default: - throw newUnexpectedArgumentException(key, DOT_PRODUCT, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); + throw newUnexpectedArgumentException(key, DOT_PRODUCT, NEAREST_NEIGHBOR, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); } } @@ -403,17 +425,97 @@ public class SelectParser implements Parser { return orItem; } + private Item buildGeoLocation(String key, Inspector value) { + HashMap<Integer, Inspector> children = childMap(value); + Preconditions.checkArgument(children.size() == 4, "Expected 4 arguments, got %s.", children.size()); + String field = children.get(0).asString(); + var arg1 = children.get(1); + var arg2 = children.get(2); + var arg3 = children.get(3); + var loc = new Location(); + if (arg3.type() != Type.STRING) { + throw new IllegalArgumentException("Invalid geoLocation radius type "+arg3.type()+" for "+arg3); + } + double radius = DistanceParser.parse(arg3.asString()); + if (arg1.type() == Type.STRING && arg2.type() == Type.STRING) { + var c1input = children.get(1).asString(); + var c2input = children.get(2).asString(); + var coord_1 = ParsedDegree.fromString(c1input, true, false); + var coord_2 = ParsedDegree.fromString(c2input, false, true); + if (coord_1.isLatitude && coord_2.isLongitude) { + loc.setGeoCircle(coord_1.degrees, coord_2.degrees, radius); + } else if (coord_2.isLatitude && coord_1.isLongitude) { + loc.setGeoCircle(coord_2.degrees, coord_1.degrees, radius); + } else { + throw new IllegalArgumentException("Invalid geoLocation coordinates '"+c1input+"' and '"+c2input+"'"); + } + } else if (arg1.type() == Type.DOUBLE && arg2.type() == Type.DOUBLE) { + loc.setGeoCircle(arg1.asDouble(), arg2.asDouble(), radius); + } else { + throw new IllegalArgumentException("Invalid geoLocation coordinate types "+arg1.type()+" and "+arg2.type()); + } + var item = new GeoLocationItem(loc, field); + Inspector annotations = getAnnotations(value); + if (annotations != null){ + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (LABEL.equals(annotation_name)) { + item.setLabel(annotation_value.asString()); + } + }); + } + return item; + } + + private Item buildNearestNeighbor(String key, Inspector value) { + HashMap<Integer, Inspector> children = childMap(value); + Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); + String field = children.get(0).asString(); + String property = children.get(1).asString(); + NearestNeighborItem item = new NearestNeighborItem(field, property); + Inspector annotations = getAnnotations(value); + if (annotations != null){ + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (TARGET_HITS.equals(annotation_name)){ + item.setTargetNumHits((int)(annotation_value.asDouble())); + } + if (TARGET_NUM_HITS.equals(annotation_name)){ + item.setTargetNumHits((int)(annotation_value.asDouble())); + } + if (DISTANCE_THRESHOLD.equals(annotation_name)) { + double distanceThreshold = annotation_value.asDouble(); + item.setDistanceThreshold(distanceThreshold); + } + if (HNSW_EXPLORE_ADDITIONAL_HITS.equals(annotation_name)) { + int hnswExploreAdditionalHits = (int)(annotation_value.asDouble()); + item.setHnswExploreAdditionalHits(hnswExploreAdditionalHits); + } + if (APPROXIMATE.equals(annotation_name)) { + boolean allowApproximate = annotation_value.asBool(); + item.setAllowApproximate(allowApproximate); + } + if (LABEL.equals(annotation_name)) { + item.setLabel(annotation_value.asString()); + } + }); + } + return item; + } + + @SuppressWarnings("deprecation") private CompositeItem buildWeakAnd(String key, Inspector value) { WeakAndItem weakAnd = new WeakAndItem(); addItemsFromInspector(weakAnd, value); Inspector annotations = getAnnotations(value); - if (annotations != null){ + if (annotations != null) { annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { - if (TARGET_NUM_HITS.equals(annotation_name)){ + if (TARGET_HITS.equals(annotation_name)){ weakAnd.setN((int)(annotation_value.asDouble())); } - if (SCORE_THRESHOLD.equals(annotation_name)){ + if (TARGET_NUM_HITS.equals(annotation_name)) { + weakAnd.setN((int)(annotation_value.asDouble())); + } + if (SCORE_THRESHOLD.equals(annotation_name)) { weakAnd.setScoreThreshold((int)(annotation_value.asDouble())); } }); @@ -662,7 +764,10 @@ public class SelectParser implements Parser { HashMap<Integer, Inspector> children = childMap(value); Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); - Integer target_num_hits= getIntegerAnnotation(TARGET_NUM_HITS, annotations, DEFAULT_TARGET_NUM_HITS); + Integer target_num_hits= getIntegerAnnotation(TARGET_HITS, annotations, null); + if (target_num_hits == null) { + target_num_hits= getIntegerAnnotation(TARGET_NUM_HITS, annotations, DEFAULT_TARGET_NUM_HITS); + } WandItem out = new WandItem(children.get(0).asString(), target_num_hits); @@ -1097,9 +1202,10 @@ public class SelectParser implements Parser { private void connectItems() { for (ConnectedItem entry : connectedItems) { TaggableItem to = identifiedItems.get(entry.toId); - Preconditions.checkNotNull(to, - "Item '%s' was specified to connect to item with ID %s, which does not " - + "exist in the query.", entry.fromItem, entry.toId); + if (to == null) + throw new IllegalArgumentException("Item '" + entry.fromItem + + "' was specified to connect to item with ID " + entry.toId + + ", which does not exist in the query."); entry.fromItem.setConnectivity((Item) to, entry.weight); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/Sorting.java b/container-search/src/main/java/com/yahoo/search/query/Sorting.java index 6518dcd5b6d..0a0ae65d524 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Sorting.java +++ b/container-search/src/main/java/com/yahoo/search/query/Sorting.java @@ -3,6 +3,7 @@ package com.yahoo.search.query; import com.ibm.icu.text.Collator; import com.ibm.icu.util.ULocale; +import com.yahoo.processing.IllegalInputException; import com.yahoo.text.Utf8; import java.nio.ByteBuffer; @@ -88,7 +89,7 @@ public class Sorting implements Cloneable { } else if (STRENGTH_IDENTICAL.equalsIgnoreCase(s)) { strength = UcaSorter.Strength.IDENTICAL; } else { - throw new IllegalArgumentException("Unknown collation strength: '" + s + "'"); + throw new IllegalInputException("Unknown collation strength: '" + s + "'"); } sorter = new UcaSorter(sortString.substring(startPar+1, commaPos), sortString.substring(commaPos+1, commaopt), strength); } else { @@ -99,9 +100,9 @@ public class Sorting implements Cloneable { } } else { if (funcName.isEmpty()) { - throw new IllegalArgumentException("No sort function specified"); + throw new IllegalInputException("No sort function specified"); } else { - throw new IllegalArgumentException("Unknown sort function '" + funcName + "'"); + throw new IllegalInputException("Unknown sort function '" + funcName + "'"); } } } else { @@ -196,7 +197,7 @@ public class Sorting implements Cloneable { if (legalAttributeName.matcher(fieldName).matches()) { this.fieldName = fieldName; } else { - throw new IllegalArgumentException("Illegal attribute name '" + fieldName + "' for sorting. Requires '" + legalAttributeName.pattern() + "'"); + throw new IllegalInputException("Illegal attribute name '" + fieldName + "' for sorting. Requires '" + legalAttributeName.pattern() + "'"); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java index 1ba30275dc1..4fb9e1e9afa 100644 --- a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java +++ b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java @@ -38,10 +38,10 @@ public class QueryContext implements Cloneable { owner.getModel().getExecution().trace().trace(message,traceLevel); } /** - * Adds a key-value which will be logged to the access log for this query (by doing toString() on the value + * Adds a key-value which will be logged to the access log for this query (by doing toString() on the value). * Multiple values may be set to the same key. A value cannot be removed once set. */ - public void logValue(String key,Object value) { + public void logValue(String key, Object value) { owner.getModel().getExecution().trace().logValue(key, value.toString()); } diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java index 9e53f9d8ea9..df96d314455 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java @@ -4,8 +4,7 @@ package com.yahoo.search.query.parser; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.parser.SpecialTokenRegistry; -import com.yahoo.prelude.query.parser.SpecialTokens; +import com.yahoo.language.process.SpecialTokens; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; @@ -19,7 +18,7 @@ public final class ParserEnvironment { private IndexFacts indexFacts = new IndexFacts(); private Linguistics linguistics = new SimpleLinguistics(); - private SpecialTokens specialTokens = new SpecialTokens(); + private SpecialTokens specialTokens = SpecialTokens.empty(); public IndexFacts getIndexFacts() { return indexFacts; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllReferencesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllReferencesQueryProfileVisitor.java deleted file mode 100644 index eda8bf78b68..00000000000 --- a/container-search/src/main/java/com/yahoo/search/query/profile/AllReferencesQueryProfileVisitor.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.query.profile; - -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.QueryProfileFieldType; -import com.yahoo.search.query.profile.types.QueryProfileType; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -/** - * @author bratseth - */ -final class AllReferencesQueryProfileVisitor extends PrefixQueryProfileVisitor { - - /** A map of query profile types */ - private Set<CompoundName> references = new HashSet<>(); - - public AllReferencesQueryProfileVisitor(CompoundName prefix) { - super(prefix); - } - - @Override - public void onValue(String name, Object value, - DimensionBinding binding, - QueryProfile owner, - DimensionValues variant) {} - - @Override - public void onQueryProfileInsidePrefix(QueryProfile profile, - DimensionBinding binding, - QueryProfile owner, - DimensionValues variant) { - references.add(currentPrefix); - } - - /** Returns the values resulting from this visiting */ - public Set<CompoundName> getResult() { return references; } - - /** Returns false - we are not done until we have seen all */ - public boolean isDone() { return false; } - -} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllTypesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllTypesQueryProfileVisitor.java deleted file mode 100644 index 6bf17d70c70..00000000000 --- a/container-search/src/main/java/com/yahoo/search/query/profile/AllTypesQueryProfileVisitor.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.query.profile; - -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.QueryProfileFieldType; -import com.yahoo.search.query.profile.types.QueryProfileType; - -import java.util.HashMap; -import java.util.Map; - -/** - * @author bratseth - */ -final class AllTypesQueryProfileVisitor extends PrefixQueryProfileVisitor { - - /** A map of query profile types */ - private Map<CompoundName, QueryProfileType> types = new HashMap<>(); - - public AllTypesQueryProfileVisitor(CompoundName prefix) { - super(prefix); - } - - @Override - public void onValue(String name, Object value, - DimensionBinding binding, - QueryProfile owner, - DimensionValues variant) {} - - - @Override - public void onQueryProfileInsidePrefix(QueryProfile profile, - DimensionBinding binding, - QueryProfile owner, - DimensionValues variant) { - if (profile.getType() != null) - addReachableTypes(currentPrefix, profile.getType()); - } - - private void addReachableTypes(CompoundName name, QueryProfileType type) { - types.putIfAbsent(name, type); // Types visited earlier has precedence: profile.type overrides profile.inherited.type - for (FieldDescription fieldDescription : type.fields().values()) { - if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) continue; - QueryProfileFieldType fieldType = (QueryProfileFieldType)fieldDescription.getType(); - if (fieldType.getQueryProfileType() !=null) { - addReachableTypes(name.append(fieldDescription.getName()), fieldType.getQueryProfileType()); - } - } - } - - /** Returns the values resulting from this visiting */ - public Map<CompoundName, QueryProfileType> getResult() { return types; } - - /** Returns false - we are not done until we have seen all */ - public boolean isDone() { return false; } - -} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllUnoverridableQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllUnoverridableQueryProfileVisitor.java deleted file mode 100644 index 4bae6823500..00000000000 --- a/container-search/src/main/java/com/yahoo/search/query/profile/AllUnoverridableQueryProfileVisitor.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.query.profile; - -import com.yahoo.processing.request.CompoundName; - -import java.util.HashSet; -import java.util.Set; - -/** - * @author bratseth - */ -final class AllUnoverridableQueryProfileVisitor extends PrefixQueryProfileVisitor { - - /** A map of query profile types */ - private Set<CompoundName> unoverridables = new HashSet<>(); - - public AllUnoverridableQueryProfileVisitor(CompoundName prefix) { - super(prefix); - } - - @Override - public void onValue(String name, Object value, - DimensionBinding binding, - QueryProfile owner, - DimensionValues variant) { - addUnoverridable(name, currentPrefix.append(name), binding, owner); - } - - @Override - public void onQueryProfileInsidePrefix(QueryProfile profile, - DimensionBinding binding, - QueryProfile owner, - DimensionValues variant) { - addUnoverridable(currentPrefix.last(), currentPrefix, binding, owner); - } - - private void addUnoverridable(String localName, - CompoundName fullName, - DimensionBinding binding, - QueryProfile owner) { - if (owner == null) return; - - Boolean isOverridable = owner.isLocalOverridable(localName, binding); - if (isOverridable != null && ! isOverridable) - unoverridables.add(fullName); - } - - /** Returns the values resulting from this visiting */ - public Set<CompoundName> getResult() { return unoverridables; } - - /** Returns false - we are not done until we have seen all */ - public boolean isDone() { return false; } - -} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java index f27500085e1..b24bf1195eb 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java @@ -13,11 +13,11 @@ import java.util.Map; */ final class AllValuesQueryProfileVisitor extends PrefixQueryProfileVisitor { - private Map<String, ValueWithSource> values = new HashMap<>(); + private final Map<String, ValueWithSource> values = new HashMap<>(); /* Lists all values starting at prefix */ - public AllValuesQueryProfileVisitor(CompoundName prefix) { - super(prefix); + public AllValuesQueryProfileVisitor(CompoundName prefix, CompoundNameChildCache pathCache) { + super(prefix, pathCache); } @Override @@ -26,7 +26,7 @@ final class AllValuesQueryProfileVisitor extends PrefixQueryProfileVisitor { DimensionBinding binding, QueryProfile owner, DimensionValues variant) { - putValue(localName, value, owner, variant); + putValue(localName, value, null, owner, variant, binding); } @Override @@ -34,17 +34,29 @@ final class AllValuesQueryProfileVisitor extends PrefixQueryProfileVisitor { DimensionBinding binding, QueryProfile owner, DimensionValues variant) { - putValue("", profile.getValue(), owner, variant); + putValue("", profile.getValue(), profile, owner, variant, binding); } - private void putValue(String key, Object value, QueryProfile owner, DimensionValues variant) { - if (value == null) return; - CompoundName fullName = currentPrefix.append(key); - if (fullName.isEmpty()) return; // Avoid putting a non-leaf (subtree) root in the list - if (values.containsKey(fullName.toString())) return; // The first value encountered has priority + private void putValue(String key, + Object value, + QueryProfile profile, + QueryProfile owner, + DimensionValues variant, + DimensionBinding binding) { + CompoundName fullName = cache.append(currentPrefix, key); + + ValueWithSource existing = values.get(fullName.toString()); + + // The first value encountered has priority and values have priority over profiles + if (existing != null && (existing.value() != null || value == null)) return; + + Boolean isOverridable = owner != null ? owner.isLocalOverridable(key, binding) : null; values.put(fullName.toString(), new ValueWithSource(value, owner == null ? "anonymous" : owner.getSource(), + isOverridable != null && ! isOverridable, + profile != null, + profile == null ? null : profile.getType(), variant)); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java index 11864e60cec..9eab6629a45 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java @@ -22,7 +22,7 @@ import java.util.Map; public class BackedOverridableQueryProfile extends OverridableQueryProfile implements Cloneable { /** The backing read only query profile, or null if this is not backed */ - private QueryProfile backingProfile; + private final QueryProfile backingProfile; /** * Creates an overridable profile from the given backing profile. The backing profile will never be @@ -31,7 +31,7 @@ public class BackedOverridableQueryProfile extends OverridableQueryProfile imple * @param backingProfile the backing profile, which is assumed read only, never null */ public BackedOverridableQueryProfile(QueryProfile backingProfile) { - Validator.ensureNotNull("An overridable query profile must be backed by a real query profile",backingProfile); + Validator.ensureNotNull("An overridable query profile must be backed by a real query profile", backingProfile); setType(backingProfile.getType()); this.backingProfile = backingProfile; } @@ -49,7 +49,7 @@ public class BackedOverridableQueryProfile extends OverridableQueryProfile imple protected Object localLookup(String localName, DimensionBinding dimensionBinding) { Object valueInThis = super.localLookup(localName, dimensionBinding); if (valueInThis != null) return valueInThis; - return backingProfile.localLookup(localName, dimensionBinding); + return backingProfile.localLookup(localName, dimensionBinding.createFor(backingProfile.getDimensions())); } protected Boolean isLocalInstanceOverridable(String localName) { @@ -88,23 +88,23 @@ public class BackedOverridableQueryProfile extends OverridableQueryProfile imple } @Override - protected void visitVariants(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding) { + protected void visitVariants(boolean allowContent, QueryProfileVisitor visitor, DimensionBinding dimensionBinding) { super.visitVariants(allowContent, visitor, dimensionBinding); if (visitor.isDone()) return; - backingProfile.visitVariants(allowContent, visitor, dimensionBinding); + backingProfile.visitVariants(allowContent, visitor, dimensionBinding.createFor(backingProfile.getDimensions())); } @Override - protected void visitInherited(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) { - super.visitInherited(allowContent,visitor,dimensionBinding, owner); + protected void visitInherited(boolean allowContent, QueryProfileVisitor visitor, DimensionBinding dimensionBinding, QueryProfile owner) { + super.visitInherited(allowContent, visitor, dimensionBinding, owner); if (visitor.isDone()) return; - backingProfile.visitInherited(allowContent,visitor,dimensionBinding,owner); + backingProfile.visitInherited(allowContent, visitor, dimensionBinding.createFor(backingProfile.getDimensions()), owner); } /** Returns a value from the content of this: The value in this, or the value from the backing if not set in this */ protected Object getContent(String localKey) { - Object value=super.getContent(localKey); - if (value!=null) return value; + Object value = super.getContent(localKey); + if (value != null) return value; return backingProfile.getContent(localKey); } @@ -113,19 +113,19 @@ public class BackedOverridableQueryProfile extends OverridableQueryProfile imple * All the values in this, and all values in the backing where an overriding value is not set in this */ @Override - protected Map<String,Object> getContent() { - Map<String,Object> thisContent=super.getContent(); - Map<String,Object> backingContent=backingProfile.getContent(); + protected Map<String, Object> getContent() { + Map<String,Object> thisContent = super.getContent(); + Map<String,Object> backingContent = backingProfile.getContent(); if (thisContent.isEmpty()) return backingContent; // Shortcut if (backingContent.isEmpty()) return thisContent; // Shortcut - Map<String,Object> content=new HashMap<>(backingContent); + Map<String, Object> content = new HashMap<>(backingContent); content.putAll(thisContent); return content; } @Override public String toString() { - return "overridable wrapper of " + backingProfile.toString(); + return "overridable wrapper of " + backingProfile; } @Override diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/CompoundNameChildCache.java b/container-search/src/main/java/com/yahoo/search/query/profile/CompoundNameChildCache.java new file mode 100644 index 00000000000..4163e45ae61 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/CompoundNameChildCache.java @@ -0,0 +1,24 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.processing.request.CompoundName; + +import java.util.HashMap; +import java.util.Map; + +/** + * Cache for compound names created through {@link CompoundName#append(String)}. + * Creating new {@link CompoundName}s can be expensive, and since they are immutable, they + * are safe to cache and reuse. Use this if you will create <em>a lot</em> of them, by appending suffixes. + * + * @author jonmv + */ +public final class CompoundNameChildCache { + + private final Map<CompoundName, Map<String, CompoundName>> cache = new HashMap<>(); + + public CompoundName append(CompoundName prefix, String suffix) { + return cache.computeIfAbsent(prefix, __ -> new HashMap<>()).computeIfAbsent(suffix, prefix::append); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java index 50bd2c58da8..43462e8f327 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java @@ -2,11 +2,9 @@ package com.yahoo.search.query.profile; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; /** * An immutable, binding of a list of dimensions to dimension values @@ -22,25 +20,25 @@ public class DimensionBinding { private DimensionValues values; /** The binding from those dimensions to values, and possibly other values */ - private Map<String, String> context; // TODO: This is not needed any more + private Map<String, String> context; public static final DimensionBinding nullBinding = - new DimensionBinding(Collections.unmodifiableList(Collections.emptyList()), DimensionValues.empty, null); + new DimensionBinding(List.of(), DimensionValues.empty, null); public static final DimensionBinding invalidBinding = - new DimensionBinding(Collections.unmodifiableList(Collections.emptyList()), DimensionValues.empty, null); + new DimensionBinding(List.of(), DimensionValues.empty, null); /** Whether the value array contains only nulls */ - private boolean containsAllNulls; + private final boolean containsAllNulls; // NOTE: Map must be ordered - public static DimensionBinding createFrom(Map<String,String> values) { + public static DimensionBinding createFrom(Map<String, String> values) { return createFrom(new ArrayList<>(values.keySet()), values); } /** Creates a binding from a variant and a context. Any of the arguments may be null. */ // NOTE: Map must be ordered - public static DimensionBinding createFrom(List<String> dimensions, Map<String,String> context) { + public static DimensionBinding createFrom(List<String> dimensions, Map<String, String> context) { if (dimensions == null || dimensions.size() == 0) { if (context == null) return nullBinding; if (dimensions == null) return new DimensionBinding(null, DimensionValues.empty, context); // Null, but must preserve context @@ -51,7 +49,7 @@ public class DimensionBinding { /** Creates a binding from a variant and a context. Any of the arguments may be null. */ public static DimensionBinding createFrom(List<String> dimensions, DimensionValues dimensionValues) { - if (dimensionValues==null || dimensionValues == DimensionValues.empty) return nullBinding; + if (dimensionValues == null || dimensionValues == DimensionValues.empty) return nullBinding; // If null, preserve raw material for creating a context later (in createFor) if (dimensions == null) return new DimensionBinding(null, dimensionValues, null); @@ -61,14 +59,13 @@ public class DimensionBinding { /** Returns a binding for a (possibly) new set of variants. Variants may be null, but not bindings */ public DimensionBinding createFor(List<String> newDimensions) { - if (newDimensions==null) return this; // Note: Not necessarily null - if no new variants then keep the existing binding - // if (this.context==null && values.length==0) return nullBinding; // No data from which to create a non-null binding - if (this.dimensions==newDimensions) return this; // Avoid creating a new object if the dimensions are the same - - Map<String,String> context=this.context; - if (context==null) - context=this.values.asContext(this.dimensions !=null ? this.dimensions : newDimensions); - return new DimensionBinding(newDimensions,extractDimensionValues(newDimensions,context),context); + if (newDimensions == null) return this; // Note: Not necessarily null - if no new variants then keep the existing binding + if (this.dimensions == newDimensions) return this; // Avoid creating a new object if the dimensions are the same + + Map<String,String> context = this.context; + if (context == null) + context = this.values.asContext(this.dimensions != null ? this.dimensions : newDimensions); + return new DimensionBinding(newDimensions, extractDimensionValues(newDimensions, context), context); } /** @@ -76,20 +73,20 @@ public class DimensionBinding { * The array will not be modified. The context is needed in order to convert this binding to another * given another set of variant dimensions. */ - private DimensionBinding(List<String> dimensions, DimensionValues values, Map<String,String> context) { - this.dimensions=dimensions; - this.values=values; + private DimensionBinding(List<String> dimensions, DimensionValues values, Map<String, String> context) { + this.dimensions = dimensions; + this.values = values; this.context = context; - containsAllNulls=values.isEmpty(); + containsAllNulls = values.isEmpty(); } /** Returns a read-only list of the dimensions of this. This value is undefined if this isNull() */ public List<String> getDimensions() { return dimensions; } /** Returns a context created from the dimensions and values of this */ - public Map<String,String> getContext() { - if (context !=null) return context; - context =values.asContext(dimensions); + public Map<String, String> getContext() { + if (context != null) return context; + context = values.asContext(dimensions); return context; } @@ -102,7 +99,7 @@ public class DimensionBinding { public DimensionValues getValues() { return values; } /** Returns true only if this binding is null (contains no values for its dimensions (if any) */ - public boolean isNull() { return dimensions==null || containsAllNulls; } + public boolean isNull() { return dimensions == null || containsAllNulls; } /** * Returns an array of the dimension values corresponding to the dimensions of this from the given context, @@ -110,10 +107,10 @@ public class DimensionBinding { * Dimensions which are not set in this context get a null value. */ private static DimensionValues extractDimensionValues(List<String> dimensions, Map<String,String> context) { - String[] dimensionValues=new String[dimensions.size()]; - if (context==null || context.size()==0) return DimensionValues.createFrom(dimensionValues); - for (int i=0; i<dimensions.size(); i++) - dimensionValues[i]=context.get(dimensions.get(i)); + String[] dimensionValues = new String[dimensions.size()]; + if (context == null || context.size() == 0) return DimensionValues.createFrom(dimensionValues); + for (int i = 0; i < dimensions.size(); i++) + dimensionValues[i] = context.get(dimensions.get(i)); return DimensionValues.createFrom(dimensionValues); } @@ -127,86 +124,57 @@ public class DimensionBinding { * * @return the combined binding, or the special invalidBinding if these two bindings are incompatible */ - public DimensionBinding combineWith(DimensionBinding binding) { - List<String> combinedDimensions = combineDimensions(getDimensions(), binding.getDimensions()); - if (combinedDimensions == null) return invalidBinding; - - // not runtime, so assume we don't need to preserve values outside the dimensions - Map<String, String> combinedValues = combineValues(getContext(), binding.getContext()); - if (combinedValues == null) return invalidBinding; - - return DimensionBinding.createFrom(combinedDimensions, combinedValues); - } - - /** Returns the binding of this (dimension->value) as a map */ - private Map<String, String> asMap() { - Map<String, String> map = new LinkedHashMap<>(); - for (int i = 0; i < Math.min(dimensions.size(), values.size()); i++) { - if (values.getValues()[i] != null) - map.put(dimensions.get(i), values.getValues()[i]); - } - return map; - } - - /** - * Returns a combined list of dimensions from two separate lists, - * or null if they are incompatible. - * This is to combine two lists to one such that the partial order in both is preserved - * (or return null if impossible). - */ - private List<String> combineDimensions(List<String> d1, List<String> d2) { - List<String> combined = new ArrayList<>(); - int d1Index = 0, d2Index=0; - while (d1Index < d1.size() && d2Index < d2.size()) { - if (d1.get(d1Index).equals(d2.get(d2Index))) { // agreement on next element - combined.add(d1.get(d1Index)); - d1Index++; - d2Index++; + public DimensionBinding combineWith(DimensionBinding other) { + List<String> d1 = getDimensions(); + List<String> d2 = other.getDimensions(); + DimensionValues v1 = getValues(); + DimensionValues v2 = other.getValues(); + List<String> dimensions = new ArrayList<>(); + List<String> values = new ArrayList<>(); + int i1 = 0, i2 = 0; + while (i1 < d1.size() && i2 < d2.size()) { + if (d1.get(i1).equals(d2.get(i2))) { // agreement on next dimension + String s1 = v1.get(i1), s2 = v2.get(i2); + if (s1 == null) + values.add(s2); + else if (s2 == null || s1.equals(s2)) + values.add(s1); + else + return invalidBinding; // disagreement on next value + + dimensions.add(d1.get(i1)); + i1++; + i2++; } - else if ( ! d2.contains(d1.get(d1Index))) { // next in d1 is independent from d2 - combined.add(d1.get(d1Index++)); + else if ( ! d2.contains(d1.get(i1))) { // next dimension in d1 is independent from d2 + dimensions.add(d1.get(i1)); + values.add(v1.get(i1)); + i1++; } - else if ( ! d1.contains(d2.get(d2Index))) { // next in d2 is independent from d1 - combined.add(d2.get(d2Index++)); + else if ( ! d1.contains(d2.get(i2))) { // next dimension in d2 is independent from d1 + dimensions.add(d2.get(i2)); + values.add(v2.get(i2)); + i2++; } else { - return null; // no independent and no agreement + return invalidBinding; // not independent and no agreement } } - if (d1Index < d1.size()) - combined.addAll(d1.subList(d1Index, d1.size())); - else if (d2Index < d2.size()) - combined.addAll(d2.subList(d2Index, d2.size())); - - return combined; - } - - /** - * Returns a combined map of dimension values from two separate maps, - * or null if they are incompatible. - */ - private Map<String, String> combineValues(Map<String, String> m1, Map<String, String> m2) { - Map<String, String> combinedValues = new LinkedHashMap<>(m1); - for (Map.Entry<String, String> m2Entry : m2.entrySet()) { - if (m2Entry.getValue() == null) continue; - String m1Value = m1.get(m2Entry.getKey()); - if (m1Value != null && ! m1Value.equals(m2Entry.getValue())) - return null; // conflicting values of a key - combinedValues.put(m2Entry.getKey(), m2Entry.getValue()); + while (i1 < d1.size()) { + dimensions.add(d1.get(i1)); + values.add(v1.get(i1)); + i1++; + } + while (i2 < d2.size()) { + dimensions.add(d2.get(i2)); + values.add(v2.get(i2)); + i2++; } - return combinedValues; - } - private boolean intersects(List<String> l1, List<String> l2) { - for (String l1Item : l1) - if (l2.contains(l1Item)) - return true; - return false; + return DimensionBinding.createFrom(dimensions, DimensionValues.createFrom(values.toArray(new String[0]))); } - /** - * Returns true if <code>this == invalidBinding</code> - */ + /** Returns true if this == invalidBinding */ public boolean isInvalid() { return this == invalidBinding; } @Override @@ -226,7 +194,7 @@ public class DimensionBinding { /** Two bindings are equal if they contain the same dimensions and the same non-null values */ @Override public boolean equals(Object o) { - if (o==this) return true; + if (o == this) return true; if (! (o instanceof DimensionBinding)) return false; DimensionBinding other = (DimensionBinding)o; if ( ! this.dimensions.equals(other.dimensions)) return false; @@ -236,7 +204,7 @@ public class DimensionBinding { @Override public int hashCode() { - return dimensions.hashCode() + 17 * values.hashCode(); + return Objects.hash(dimensions, values); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java index 9eb50c0f72e..f3c4548c491 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java @@ -21,7 +21,7 @@ public class DimensionValues implements Comparable<DimensionValues> { public static final DimensionValues empty = new DimensionValues(new String[] {}); public static DimensionValues createFrom(String[] values) { - if (values==null || values.length==0 || containsAllNulls(values)) return empty; + if (values == null || values.length == 0 || containsAllNulls(values)) return empty; return new DimensionValues(values); } @@ -34,10 +34,10 @@ public class DimensionValues implements Comparable<DimensionValues> { */ private DimensionValues(String[] values) { if (values == null) throw new NullPointerException("Dimension values cannot be null"); - this.values=Arrays.copyOf(values, values.length); + this.values = Arrays.copyOf(values, values.length); } - /** Returns true if this is has the same value every place it has a value as the givenValues. */ + /** Returns true if this is has the same value every place it has a value as the given values. */ public boolean matches(DimensionValues givenValues) { for (int i = 0; i < this.size() || i < givenValues.size() ; i++) if ( ! matches(this.get(i), givenValues.get(i))) @@ -73,14 +73,14 @@ public class DimensionValues implements Comparable<DimensionValues> { /** Helper method which uses compareTo to return whether this is most specific */ public boolean isMoreSpecificThan(DimensionValues other) { - return this.compareTo(other)<0; + return this.compareTo(other) < 0; } @Override public boolean equals(Object o) { if (this == o) return true; if ( ! (o instanceof DimensionValues)) return false; - DimensionValues other = (DimensionValues)o; + DimensionValues other = (DimensionValues) o; for (int i = 0; i < this.size() || i < other.size(); i++) { if (get(i) == null) { if (other.get(i) != null) return false; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java index 4dc9ade62e5..e32d2dc226d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java @@ -22,8 +22,7 @@ public class ModelObjectMap extends PropertyMap { */ @Override protected boolean shouldSet(CompoundName name, Object value) { - if (value == null) return true; - return ! FieldType.isLegalFieldValue(value); + return value != null && ! FieldType.isLegalFieldValue(value); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/PrefixQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/PrefixQueryProfileVisitor.java index 690a48f8124..b53fc4f96f2 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/PrefixQueryProfileVisitor.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/PrefixQueryProfileVisitor.java @@ -3,6 +3,11 @@ package com.yahoo.search.query.profile; import com.yahoo.processing.request.CompoundName; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashMap; +import java.util.Map; + /** * A query profile visitor which keeps track of name prefixes and can skip values outside a given prefix * @@ -10,18 +15,22 @@ import com.yahoo.processing.request.CompoundName; */ abstract class PrefixQueryProfileVisitor extends QueryProfileVisitor { + protected final CompoundNameChildCache cache; + /** Only call onValue/onQueryProfile for nodes having this prefix */ private final CompoundName prefix; /** The current prefix, relative to prefix. */ protected CompoundName currentPrefix = CompoundName.empty; + private final Deque<CompoundName> currentPrefixes = new ArrayDeque<>(); private int prefixComponentIndex = -1; - public PrefixQueryProfileVisitor(CompoundName prefix) { + public PrefixQueryProfileVisitor(CompoundName prefix, CompoundNameChildCache cache) { if (prefix == null) prefix = CompoundName.empty; this.prefix = prefix; + this.cache = cache; } @Override @@ -40,18 +49,19 @@ abstract class PrefixQueryProfileVisitor extends QueryProfileVisitor { @Override public final boolean enter(String name) { - prefixComponentIndex++; - if (prefixComponentIndex-1 < prefix.size()) return true; // we're in the given prefix, which should not be included in the name - currentPrefix = currentPrefix.append(name); + if (prefixComponentIndex++ < prefix.size()) return true; // we're in the given prefix, which should not be included in the name + if ( ! name.isEmpty()) { + currentPrefixes.push(currentPrefix); + currentPrefix = cache.append(currentPrefix, name); + } return true; } @Override public final void leave(String name) { - prefixComponentIndex--; - if (prefixComponentIndex < prefix.size()) return; // we're in the given prefix, which should not be included in the name - if ( ! name.isEmpty() && ! currentPrefix.isEmpty()) - currentPrefix = currentPrefix.first(currentPrefix.size() - 1); + if (--prefixComponentIndex < prefix.size()) return; // we're in the given prefix, which should not be included in the name + if ( ! name.isEmpty()) + currentPrefix = currentPrefixes.pop(); } /** diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java index 7ae18f96d86..d3da2fd076a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java @@ -4,6 +4,7 @@ package com.yahoo.search.query.profile; import com.google.common.collect.ImmutableList; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.FreezableSimpleComponent; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.processing.request.Properties; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; @@ -17,7 +18,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -146,7 +146,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * Returns the content fields declared in this (i.e not including those inherited) as a read-only map. * @throws IllegalStateException if this is frozen */ - public Map<String,Object> declaredContent() { + public Map<String, Object> declaredContent() { ensureNotFrozen(); return content.unmodifiableMap(); } @@ -202,6 +202,14 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } /** + * Sets the overridability of a field in this profile, + * this overrides the corresponding setting in the type (if any) + */ + public final void setOverridable(String fieldName, boolean overridable, DimensionValues dimensionValues) { + setOverridable(new CompoundName(fieldName), overridable, DimensionBinding.createFrom(getDimensions(), dimensionValues)); + } + + /** * Return all objects that start with the given prefix path using no context. Use "" to list all. * <p> * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") @@ -233,7 +241,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") * will return {"d" => "a.d-value","e" => "a.e-value"} */ - public final Map<String, Object> listValues(CompoundName prefix, Map<String,String> context) { + public final Map<String, Object> listValues(CompoundName prefix, Map<String, String> context) { return listValues(prefix, context, null); } @@ -257,42 +265,14 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } AllValuesQueryProfileVisitor visitValues(CompoundName prefix, Map<String, String> context) { - DimensionBinding dimensionBinding = DimensionBinding.createFrom(getDimensions(), context); - - AllValuesQueryProfileVisitor visitor = new AllValuesQueryProfileVisitor(prefix); - accept(visitor, dimensionBinding, null); - return visitor; - } - - /** - * Lists types reachable from this, indexed by the prefix having that type. - * If this is itself typed, this' type will be included with an empty prefix - */ - public Map<CompoundName, QueryProfileType> listTypes(CompoundName prefix, Map<String, String> context) { - DimensionBinding dimensionBinding = DimensionBinding.createFrom(getDimensions(), context); - AllTypesQueryProfileVisitor visitor = new AllTypesQueryProfileVisitor(prefix); - accept(visitor, dimensionBinding, null); - return visitor.getResult(); - } - - /** - * Lists references reachable from this. - */ - Set<CompoundName> listReferences(CompoundName prefix, Map<String, String> context) { - DimensionBinding dimensionBinding = DimensionBinding.createFrom(getDimensions(),context); - AllReferencesQueryProfileVisitor visitor = new AllReferencesQueryProfileVisitor(prefix); - accept(visitor,dimensionBinding,null); - return visitor.getResult(); + return visitValues(prefix, context, new CompoundNameChildCache()); } - /** - * Lists every entry (value or reference) reachable from this which is not overridable - */ - Set<CompoundName> listUnoverridable(CompoundName prefix, Map<String, String> context) { - DimensionBinding dimensionBinding = DimensionBinding.createFrom(getDimensions(),context); - AllUnoverridableQueryProfileVisitor visitor = new AllUnoverridableQueryProfileVisitor(prefix); - accept(visitor, dimensionBinding, null); - return visitor.getResult(); + AllValuesQueryProfileVisitor visitValues(CompoundName prefix, Map<String, String> context, + CompoundNameChildCache pathCache) { + AllValuesQueryProfileVisitor visitor = new AllValuesQueryProfileVisitor(prefix, pathCache); + accept(visitor, DimensionBinding.createFrom(getDimensions(), context), null); + return visitor; } /** @@ -371,11 +351,11 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * @throws IllegalArgumentException if the given name is illegal given the types of this or any nested query profile * @throws IllegalStateException if this query profile is frozen */ - public final void set(CompoundName name, Object value, Map<String,String> context, QueryProfileRegistry registry) { + public final void set(CompoundName name, Object value, Map<String, String> context, QueryProfileRegistry registry) { set(name, value, DimensionBinding.createFrom(getDimensions(), context), registry); } - public final void set(String name, Object value, Map<String,String> context, QueryProfileRegistry registry) { + public final void set(String name, Object value, Map<String, String> context, QueryProfileRegistry registry) { set(new CompoundName(name), value, DimensionBinding.createFrom(getDimensions(), context), registry); } @@ -396,7 +376,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * @throws IllegalArgumentException if the given name is illegal given the types of this or any nested query profile * @throws IllegalStateException if this query profile is frozen */ - public final void set(String name,Object value, DimensionValues dimensionValues, QueryProfileRegistry registry) { + public final void set(String name, Object value, DimensionValues dimensionValues, QueryProfileRegistry registry) { set(new CompoundName(name), value, DimensionBinding.createFrom(getDimensions(), dimensionValues), registry); } @@ -482,7 +462,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable setNode(name, value, null, binding, registry); } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "'", e); + throw new IllegalInputException("Could not set '" + name + "' to '" + value + "'", e); } } @@ -509,9 +489,14 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable */ Boolean isLocalOverridable(String localName, DimensionBinding binding) { if (localLookup(localName, binding) == null) return null; // Not set + if ( ! binding.isNull() && getVariants() != null) { + Boolean variantIsOverriable = getVariants().isOverridable(localName, binding.getValues()); + if (variantIsOverriable != null) + return variantIsOverriable; + } Boolean isLocalInstanceOverridable = isLocalInstanceOverridable(localName); if (isLocalInstanceOverridable != null) - return isLocalInstanceOverridable.booleanValue(); + return isLocalInstanceOverridable; if (type != null) return type.isOverridable(localName); return true; } @@ -564,7 +549,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable if (visitor.isDone()) return; if (allowContent) { - visitContent(visitor,dimensionBinding); + visitContent(visitor, dimensionBinding); if (visitor.isDone()) return; } @@ -601,7 +586,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable visitor.acceptValue(contentKey, getContent(contentKey), dimensionBinding, this, null); } else { // get all content in this - for (Map.Entry<String,Object> entry : getContent().entrySet()) { + for (Map.Entry<String, Object> entry : getContent().entrySet()) { visitor.acceptValue(entry.getKey(), entry.getValue(), dimensionBinding, this, null); if (visitor.isDone()) return; } @@ -614,12 +599,12 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } /** Returns all the content from this as an unmodifiable map */ - protected Map<String,Object> getContent() { + protected Map<String, Object> getContent() { return content.unmodifiableMap(); } /** Sets the value of a node in <i>this</i> profile - the local name given must not be nested (contain dots) */ - protected QueryProfile setLocalNode(String localName, Object value,QueryProfileType parentType, + protected QueryProfile setLocalNode(String localName, Object value, QueryProfileType parentType, DimensionBinding dimensionBinding, QueryProfileRegistry registry) { if (parentType != null && type == null && ! isFrozen()) type = parentType; @@ -637,9 +622,10 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable static Object combineValues(Object newValue, Object existingValue) { if (newValue instanceof QueryProfile) { QueryProfile newProfile = (QueryProfile)newValue; - if ( existingValue == null || ! (existingValue instanceof QueryProfile)) { - if (!isModifiable(newProfile)) + if ( ! (existingValue instanceof QueryProfile)) { + if ( ! isModifiable(newProfile)) { newProfile = new BackedOverridableQueryProfile(newProfile); // Make the query profile reference overridable + } newProfile.value = existingValue; return newProfile; } @@ -717,15 +703,15 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable FieldDescription fieldDescription = type.getField(localName); if (fieldDescription == null) { if (type.isStrict()) - throw new IllegalArgumentException("'" + localName + "' is not declared in " + type + ", and the type is strict"); + throw new IllegalInputException("'" + localName + "' is not declared in " + type + ", and the type is strict"); return value; } if (registry == null && (fieldDescription.getType() instanceof QueryProfileFieldType)) - throw new IllegalArgumentException("A registry was not passed: Query profile references is not supported"); + throw new IllegalInputException("A registry was not passed: Query profile references is not supported"); Object convertedValue = fieldDescription.getType().convertFrom(value, registry); if (convertedValue == null) - throw new IllegalArgumentException("'" + value + "' is not a " + fieldDescription.getType().toInstanceDescription()); + throw new IllegalInputException("'" + value + "' is not a " + fieldDescription.getType().toInstanceDescription()); return convertedValue; } @@ -759,14 +745,19 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * Sets the overridability of a field in this profile, * this overrides the corresponding setting in the type (if any) */ - private void setOverridable(CompoundName fieldName,boolean overridable,DimensionBinding dimensionBinding) { + private void setOverridable(CompoundName fieldName, boolean overridable, DimensionBinding dimensionBinding) { QueryProfile parent = lookupParentExact(fieldName, true, dimensionBinding); - if (parent.overridable == null) - parent.overridable = new HashMap<>(); - parent.overridable.put(fieldName.last(), overridable); + if (dimensionBinding.isNull()) { + if (parent.overridable == null) + parent.overridable = new HashMap<>(); + parent.overridable.put(fieldName.last(), overridable); + } + else { + variants.setOverridable(fieldName.last(), overridable, dimensionBinding.getValues()); + } } - /** Sets a value to a (possibly non-local) node. The parent query profile holding the value is returned */ + /** Sets a value to a (possibly non-local) node. */ private void setNode(CompoundName name, Object value, QueryProfileType parentType, DimensionBinding dimensionBinding, QueryProfileRegistry registry) { ensureNotFrozen(); @@ -840,13 +831,10 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable value = convertToSubstitutionString(value); if (dimensionBinding.isNull()) { - Object combinedValue; - if (value instanceof QueryProfile) - combinedValue = combineValues(value, content == null ? null : content.get(localName)); - else - combinedValue = combineValues(value, localLookup(localName, dimensionBinding)); - - if (combinedValue!=null) + Object combinedValue = value instanceof QueryProfile + ? combineValues(value, content == null ? null : content.get(localName)) + : combineValues(value, localLookup(localName, dimensionBinding)); + if (combinedValue != null) content.put(localName, combinedValue); } else { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java index cffe941b912..29a997a75dd 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java @@ -2,15 +2,23 @@ package com.yahoo.search.query.profile; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.compiled.Binding; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.compiled.DimensionalMap; import com.yahoo.search.query.profile.compiled.ValueWithSource; import com.yahoo.search.query.profile.types.QueryProfileType; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.function.BiConsumer; import java.util.logging.Logger; /** @@ -24,34 +32,38 @@ public class QueryProfileCompiler { public static CompiledQueryProfileRegistry compile(QueryProfileRegistry input) { CompiledQueryProfileRegistry output = new CompiledQueryProfileRegistry(input.getTypeRegistry()); - for (QueryProfile inputProfile : input.allComponents()) { + for (QueryProfile inputProfile : input.allComponents()) output.register(compile(inputProfile, output)); - } return output; } public static CompiledQueryProfile compile(QueryProfile in, CompiledQueryProfileRegistry registry) { try { - DimensionalMap.Builder<CompoundName, ValueWithSource> values = new DimensionalMap.Builder<>(); - DimensionalMap.Builder<CompoundName, QueryProfileType> types = new DimensionalMap.Builder<>(); - DimensionalMap.Builder<CompoundName, Object> references = new DimensionalMap.Builder<>(); - DimensionalMap.Builder<CompoundName, Object> unoverridables = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<ValueWithSource> values = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<QueryProfileType> types = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<Object> references = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<Object> unoverridables = new DimensionalMap.Builder<>(); // Resolve values for each existing variant and combine into a single data structure Set<DimensionBindingForPath> variants = collectVariants(CompoundName.empty, in, DimensionBinding.nullBinding); variants.add(new DimensionBindingForPath(DimensionBinding.nullBinding, CompoundName.empty)); // if this contains no variants - log.fine(() -> "Compiling " + in.toString() + " having " + variants.size() + " variants"); - for (DimensionBindingForPath variant : variants) { - log.finer(() -> " Compiling variant " + variant); - for (Map.Entry<String, ValueWithSource> entry : in.visitValues(variant.path(), variant.binding().getContext()).valuesWithSource().entrySet()) { - values.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); + log.fine(() -> "Compiling " + in + " having " + variants.size() + " variants"); + + CompoundNameChildCache pathCache = new CompoundNameChildCache(); + Map<DimensionBinding, Binding> bindingCache = new HashMap<>(); + for (var variant : variants) { + log.finer(() -> "Compiling variant " + variant); + Binding variantBinding = bindingCache.computeIfAbsent(variant.binding(), Binding::createFrom); + for (var entry : in.visitValues(variant.path(), variant.binding().getContext(), pathCache).valuesWithSource().entrySet()) { + CompoundName fullName = pathCache.append(variant.path, entry.getKey()); + values.put(fullName, variantBinding, entry.getValue()); + if (entry.getValue().isUnoverridable()) + unoverridables.put(fullName, variantBinding, Boolean.TRUE); + if (entry.getValue().isQueryProfile()) + references.put(fullName, variantBinding, Boolean.TRUE); + if (entry.getValue().queryProfileType() != null) + types.put(fullName, variantBinding, entry.getValue().queryProfileType()); } - for (Map.Entry<CompoundName, QueryProfileType> entry : in.listTypes(variant.path(), variant.binding().getContext()).entrySet()) - types.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); - for (CompoundName reference : in.listReferences(variant.path(), variant.binding().getContext())) - references.put(variant.path().append(reference), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored - for (CompoundName name : in.listUnoverridable(variant.path(), variant.binding().getContext())) - unoverridables.put(variant.path().append(name), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored } return new CompiledQueryProfile(in.getId(), in.getType(), @@ -71,6 +83,7 @@ public class QueryProfileCompiler { */ private static Set<DimensionBindingForPath> collectVariants(CompoundName path, QueryProfile profile, DimensionBinding currentVariant) { Set<DimensionBindingForPath> variants = new HashSet<>(); + variants.addAll(collectVariantsFromValues(path, profile.getContent(), currentVariant)); variants.addAll(collectVariantsInThis(path, profile, currentVariant)); if (profile instanceof BackedOverridableQueryProfile) @@ -93,16 +106,31 @@ public class QueryProfileCompiler { * I.e if we have the variants [-,b=b1], [a=a1,-], [a=a2,-], * this returns the variants [a=a1,b=b1], [a=a2,b=b1] * - * This is necessary because left-specified values takes precedence, such that resolving [a=a1,b=b1] would - * lead us to the compiled profile [a=a1,-], which may contain default values for properties where + * This is necessary because left-specified values takes precedence, and resolving [a=a1,b=b1] would + * otherwise lead us to the compiled profile [a=a1,-], which may contain default values for properties where * we should have preferred variant values in [-,b=b1]. */ private static Set<DimensionBindingForPath> wildcardExpanded(Set<DimensionBindingForPath> variants) { Set<DimensionBindingForPath> expanded = new HashSet<>(); - for (var variant : variants) { - if (hasWildcardBeforeEnd(variant.binding())) - expanded.addAll(wildcardExpanded(variant, variants)); - } + + PathTree trie = new PathTree(); + for (var variant : variants) + trie.add(variant); + + // Visit all variant prefixes, grouped on path, and all their unique child bindings. + trie.forEachPrefixAndChildren((prefixes, childBindings) -> { + Set<DimensionBinding> processed = new HashSet<>(); + for (DimensionBindingForPath prefix : prefixes) + if (processed.add(prefix.binding())) // Only compute once for similar bindings, since path is equal. + if (hasWildcardBeforeEnd(prefix.binding())) + for (DimensionBinding childBinding : childBindings) + if (childBinding != prefix.binding()) { + DimensionBinding combined = prefix.binding().combineWith(childBinding); + if ( ! combined.isInvalid()) + expanded.add(new DimensionBindingForPath(combined, prefix.path())); + } + }); + return expanded; } @@ -114,20 +142,6 @@ public class QueryProfileCompiler { return false; } - private static Set<DimensionBindingForPath> wildcardExpanded(DimensionBindingForPath variantToExpand, - Set<DimensionBindingForPath> variants) { - Set<DimensionBindingForPath> expanded = new HashSet<>(); - for (var variant : variants) { - if (variant.binding().isNull()) continue; - DimensionBinding combined = variantToExpand.binding().combineWith(variant.binding()); - if ( ! combined.isInvalid() ) { - expanded.add(new DimensionBindingForPath(combined, variantToExpand.path())); - } - } - return expanded; - } - - /** Generates a set of all the (legal) combinations of the variants in the given sets */ private static Set<DimensionBindingForPath> combined(Set<DimensionBindingForPath> v1s, Set<DimensionBindingForPath> v2s) { @@ -135,7 +149,7 @@ public class QueryProfileCompiler { for (DimensionBindingForPath v1 : v1s) { if (v1.binding().isNull()) continue; for (DimensionBindingForPath v2 : v2s) { - if (v1.binding().isNull()) continue; + if (v2.binding().isNull()) continue; DimensionBinding combined = v1.binding().combineWith(v2.binding()); if ( combined.isInvalid() ) continue; @@ -157,6 +171,7 @@ public class QueryProfileCompiler { if (combinedVariant.isInvalid()) continue; // values at this point in the graph are unreachable + variants.add(new DimensionBindingForPath(combinedVariant, path)); variants.addAll(collectVariantsFromValues(path, variant.values(), combinedVariant)); for (QueryProfile variantInheritedProfile : variant.inherited()) variants.addAll(collectVariants(path, variantInheritedProfile, combinedVariant)); @@ -169,9 +184,6 @@ public class QueryProfileCompiler { Map<String, Object> values, DimensionBinding currentVariant) { Set<DimensionBindingForPath> variants = new HashSet<>(); - if ( ! values.isEmpty()) - variants.add(new DimensionBindingForPath(currentVariant, path)); // there are actual values for this variant - for (Map.Entry<String, Object> entry : values.entrySet()) { if (entry.getValue() instanceof QueryProfile) variants.addAll(collectVariants(path.append(entry.getKey()), (QueryProfile)entry.getValue(), currentVariant)); @@ -202,7 +214,7 @@ public class QueryProfileCompiler { @Override public int hashCode() { - return binding.hashCode() + 17*path.hashCode(); + return binding.hashCode() + 17 * path.hashCode(); } @Override @@ -212,4 +224,61 @@ public class QueryProfileCompiler { } + + /** + * Simple trie for CompoundName paths. + * + * @author jonmv + */ + static class PathTree { + + private final Node root = new Node(0); + + void add(DimensionBindingForPath entry) { + root.add(entry); + } + + /** Performs action on sets of path prefixes against all their (common) children. */ + void forEachPrefixAndChildren(BiConsumer<Collection<DimensionBindingForPath>, Collection<DimensionBinding>> action) { + root.visit(action); + } + + private static class Node { + + private final int depth; + private final SortedMap<String, Node> children = new TreeMap<>(); + private final List<DimensionBindingForPath> elements = new ArrayList<>(); + + private Node(int depth) { + this.depth = depth; + } + + /** Performs action on the elements of this against all child element bindings, then returns the union of these two sets. */ + Set<DimensionBinding> visit(BiConsumer<Collection<DimensionBindingForPath>, Collection<DimensionBinding>> action) { + Set<DimensionBinding> allChildren = new HashSet<>(); + for (Node child : children.values()) + allChildren.addAll(child.visit(action)); + + for (DimensionBindingForPath element : elements) + if ( ! element.binding().isNull()) + allChildren.add(element.binding()); + + action.accept(elements, allChildren); + + return allChildren; + } + + void add(DimensionBindingForPath entry) { + if (depth == entry.path().size()) + elements.add(entry); + else + children.computeIfAbsent(entry.path().get(depth), + __ -> new Node(depth + 1)).add(entry); + } + + } + + } + + } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 05e3c4fe9a0..53be827073c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -2,12 +2,15 @@ package com.yahoo.search.query.profile; import com.yahoo.collections.Pair; +import com.yahoo.language.process.Embedder; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.processing.request.properties.PropertyMap; import com.yahoo.protect.Validator; import com.yahoo.search.query.Properties; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.DimensionalValue; +import com.yahoo.search.query.profile.types.ConversionContext; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileFieldType; import com.yahoo.search.query.profile.types.QueryProfileType; @@ -27,18 +30,29 @@ import java.util.Map; public class QueryProfileProperties extends Properties { private final CompiledQueryProfile profile; + private final Embedder embedder; // Note: The priority order is: values has precedence over references /** Values which has been overridden at runtime, or null if none */ private Map<CompoundName, Object> values = null; - /** Query profile references which has been overridden at runtime, or null if none. Earlier values has precedence */ + + /** + * Query profile references which has been overridden at runtime, possibly to the null value to clear values, + * or null if none (i.e this is lazy). + * Earlier values has precedence + */ private List<Pair<CompoundName, CompiledQueryProfile>> references = null; - /** Creates an instance from a profile, throws an exception if the given profile is null */ public QueryProfileProperties(CompiledQueryProfile profile) { + this(profile, Embedder.throwsOnUse); + } + + /** Creates an instance from a profile, throws an exception if the given profile is null */ + public QueryProfileProperties(CompiledQueryProfile profile, Embedder embedder) { Validator.ensureNotNull("The profile wrapped by this cannot be null", profile); this.profile = profile; + this.embedder = embedder; } /** Returns the query profile backing this, or null if none */ @@ -49,20 +63,21 @@ public class QueryProfileProperties extends Properties { public Object get(CompoundName name, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { name = unalias(name, context); - Object value = null; - if (values != null) - value = values.get(name); - if (value == null) { - Pair<CompoundName, CompiledQueryProfile> reference = findReference(name); - if (reference != null) - return reference.getSecond().get(name.rest(reference.getFirst().size()), context, substitution); // yes; even if null + if (values != null && values.containsKey(name)) + return values.get(name); // Returns this value, even if null + + Pair<CompoundName, CompiledQueryProfile> reference = findReference(name); + if (reference != null) { + if (reference.getSecond() == null) + return null; // cleared + else + return reference.getSecond().get(name.rest(reference.getFirst().size()), context, substitution); // even if null } - if (value == null) - value = profile.get(name, context, substitution); - if (value == null) - value = super.get(name, context, substitution); - return value; + Object value = profile.get(name, context, substitution); + if (value != null) + return value; + return super.get(name, context, substitution); } /** @@ -88,28 +103,34 @@ public class QueryProfileProperties extends Properties { // Check types if ( ! profile.getTypes().isEmpty()) { - QueryProfileType type = null; + QueryProfileType type; + QueryProfileType explicitTypeFromField = null; for (int i = 0; i < name.size(); i++) { - if (type == null) // We're on the first iteration, or no type is explicitly specified + if (explicitTypeFromField != null) + type = explicitTypeFromField; + else type = profile.getType(name.first(i), context); if (type == null) continue; + String localName = name.get(i); FieldDescription fieldDescription = type.getField(localName); if (fieldDescription == null && type.isStrict()) - throw new IllegalArgumentException("'" + localName + "' is not declared in " + type + ", and the type is strict"); + throw new IllegalInputException("'" + localName + "' is not declared in " + type + ", and the type is strict"); // TODO: In addition to strictness, check legality along the way if (fieldDescription != null) { if (i == name.size() - 1) { // at the end of the path, check the assignment type - value = fieldDescription.getType().convertFrom(value, profile.getRegistry()); + value = fieldDescription.getType().convertFrom(value, new ConversionContext(profile.getRegistry(), + embedder, + context)); if (value == null) - throw new IllegalArgumentException("'" + value + "' is not a " + - fieldDescription.getType().toInstanceDescription()); + throw new IllegalInputException("'" + value + "' is not a " + + fieldDescription.getType().toInstanceDescription()); } else if (fieldDescription.getType() instanceof QueryProfileFieldType) { // If a type is specified, use that instead of the type implied by the name - type = ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); + explicitTypeFromField = ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); } } @@ -118,12 +139,12 @@ public class QueryProfileProperties extends Properties { if (value instanceof String && value.toString().startsWith("ref:")) { if (profile.getRegistry() == null) - throw new IllegalArgumentException("Runtime query profile references does not work when the " + + throw new IllegalInputException("Runtime query profile references does not work when the " + "QueryProfileProperties are constructed without a registry"); String queryProfileId = value.toString().substring(4); value = profile.getRegistry().findQueryProfile(queryProfileId); if (value == null) - throw new IllegalArgumentException("Query profile '" + queryProfileId + "' is not found"); + throw new IllegalInputException("Query profile '" + queryProfileId + "' is not found"); } if (value instanceof CompiledQueryProfile) { // this will be due to one of the two clauses above @@ -138,17 +159,31 @@ public class QueryProfileProperties extends Properties { } } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "'", e); + throw new IllegalInputException("Could not set '" + name + "' to '" + value + "'", e); } } @Override - public Map<String, Object> listProperties(CompoundName path, Map<String,String> context, + public void clearAll(CompoundName name, Map<String, String> context) { + if (references == null) + references = new ArrayList<>(); + references.add(new Pair<>(name, null)); + + if (values != null) + values.keySet().removeIf(key -> key.hasPrefix(name)); + } + + @Override + public Map<String, Object> listProperties(CompoundName path, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { path = unalias(path, context); if (context == null) context = Collections.emptyMap(); - Map<String, Object> properties = profile.listValues(path, context, substitution); + Map<String, Object> properties = new HashMap<>(); + for (var entry : profile.listValues(path, context, substitution).entrySet()) { + if (references != null && containsNullParentOf(path, references)) continue; + properties.put(entry.getKey(), entry.getValue()); + } properties.putAll(super.listProperties(path, context, substitution)); if (references != null) { @@ -165,8 +200,14 @@ public class QueryProfileProperties extends Properties { pathInReference = path.rest(refEntry.getFirst().size()); prefixToReferenceKeys = CompoundName.empty; } - for (Map.Entry<String, Object> valueEntry : refEntry.getSecond().listValues(pathInReference, context, substitution).entrySet()) { - properties.put(prefixToReferenceKeys.append(new CompoundName(valueEntry.getKey())).toString(), valueEntry.getValue()); + if (refEntry.getSecond() == null) { + if (refEntry.getFirst().hasPrefix(path)) + properties.put(prefixToReferenceKeys.toString(), null); + } + else { + for (Map.Entry<String, Object> valueEntry : refEntry.getSecond().listValues(pathInReference, context, substitution).entrySet()) { + properties.put(prefixToReferenceKeys.append(new CompoundName(valueEntry.getKey())).toString(), valueEntry.getValue()); + } } } @@ -241,6 +282,12 @@ public class QueryProfileProperties extends Properties { return null; } + private boolean containsNullParentOf(CompoundName path, List<Pair<CompoundName, CompiledQueryProfile>> properties) { + if (properties.contains(new Pair<>(path, (CompiledQueryProfile)null))) return true; + if (path.size() > 0 && containsNullParentOf(path.first(path.size() - 1), properties)) return true; + return false; + } + CompoundName unalias(CompoundName name, Map<String,String> context) { if (profile.getTypes().isEmpty()) return name; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java index 0363b50815b..eb7a6d19d91 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java @@ -62,7 +62,6 @@ public class QueryProfileRegistry extends ComponentRegistry<QueryProfile> { int slashIndex=id.getName().lastIndexOf("/"); if (slashIndex<1) return null; String parentName=id.getName().substring(0,slashIndex); - if (parentName.equals("")) return null; ComponentSpecification parentId=new ComponentSpecification(parentName,id.getVersionSpecification()); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java index 3f70ff98373..855befad658 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java @@ -16,13 +16,15 @@ public class QueryProfileVariant implements Cloneable, Comparable<QueryProfileVa private List<QueryProfile> inherited = null; - private DimensionValues dimensionValues; + private final DimensionValues dimensionValues; private Map<String, Object> values; + private final Map<String, Boolean> overridable = new HashMap<>(); + private boolean frozen = false; - private QueryProfile owner; + private final QueryProfile owner; public QueryProfileVariant(DimensionValues dimensionValues, QueryProfile owner) { this.dimensionValues = dimensionValues; @@ -35,7 +37,7 @@ public class QueryProfileVariant implements Cloneable, Comparable<QueryProfileVa * Returns the live reference to the values of this. This may be modified * if this is not frozen. */ - public Map<String,Object> values() { + public Map<String, Object> values() { if (values == null) { if (frozen) return Collections.emptyMap(); @@ -59,20 +61,26 @@ public class QueryProfileVariant implements Cloneable, Comparable<QueryProfileVa return inherited; } - public void set(String key, Object newValue) { + public Object set(String key, Object newValue) { if (values == null) values = new HashMap<>(); Object oldValue = values.get(key); - if (oldValue == null) { - values.put(key, newValue); - } else { - Object combinedOrNull = QueryProfile.combineValues(newValue, oldValue); - if (combinedOrNull != null) { - values.put(key, combinedOrNull); - } - } + Object combinedOrNull = QueryProfile.combineValues(newValue, oldValue); + if (combinedOrNull instanceof BackedOverridableQueryProfile) // Use the owner's, not the referenced dimensions + ((QueryProfile) combinedOrNull).setDimensions(owner.getDimensions().toArray(new String[0])); + if (combinedOrNull != null) + values.put(key, combinedOrNull); + return combinedOrNull; + } + + public void setOverridable(String key, boolean overridable) { + this.overridable.put(key, overridable); + } + + public Boolean isOverridable(String key) { + return overridable.get(key); } public void inherit(QueryProfile profile) { @@ -138,6 +146,7 @@ public class QueryProfileVariant implements Cloneable, Comparable<QueryProfileVa frozen=true; } + @Override public QueryProfileVariant clone() { if (frozen) return this; try { @@ -156,7 +165,7 @@ public class QueryProfileVariant implements Cloneable, Comparable<QueryProfileVa @Override public String toString() { - return "query profile variant for " + dimensionValues; + return "query profile variant of " + owner + " for " + dimensionValues; } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java index 8dedda800ea..431947b0816 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java @@ -27,10 +27,10 @@ public class QueryProfileVariants implements Freezable, Cloneable { private boolean frozen = false; /** Properties indexed by name, to support fast lookup of single values */ - private Map<String,FieldValues> fieldValuesByName = new HashMap<>(); + private Map<String, FieldValues> fieldValuesByName = new HashMap<>(); /** The inherited profiles for various dimensions settings - a set of fieldvalues of List<QueryProfile> */ - private FieldValues inheritedProfiles =new FieldValues(); + private FieldValues inheritedProfiles = new FieldValues(); /** * Field and inherited profiles sorted by specificity used for all-value visiting. @@ -43,10 +43,10 @@ public class QueryProfileVariants implements Freezable, Cloneable { * Order matters - more specific values to the left in this list are more significant than more specific values * to the right */ - private List<String> dimensions; + private final List<String> dimensions; /** The query profile this variants of */ - private QueryProfile owner; + private final QueryProfile owner; /** * Creates a set of virtual query profiles which may return varying values over the set of dimensions given. @@ -144,7 +144,7 @@ public class QueryProfileVariants implements Freezable, Cloneable { if (visitor.isDone()) return; fieldIndex++; } - else if (inheritedProfileValue != null) { // Inherited is most specific at this point + else { // Inherited is most specific at this point if (inheritedProfileValue.matches(dimensionBinding.getValues())) { @SuppressWarnings("unchecked") List<QueryProfile> inheritedProfileList = (List<QueryProfile>)inheritedProfileValue.getValue(); @@ -219,7 +219,7 @@ public class QueryProfileVariants implements Freezable, Cloneable { ensureNotFrozen(); // Update variant - getVariant(dimensionValues, true).set(fieldName, value); + Object combinedValue = getVariant(dimensionValues, true).set(fieldName, value); // Update per-variable optimized structure FieldValues fieldValues = fieldValuesByName.get(fieldName); @@ -228,12 +228,24 @@ public class QueryProfileVariants implements Freezable, Cloneable { fieldValuesByName.put(fieldName, fieldValues); } - Object combinedValue = QueryProfile.combineValues(value, fieldValues.getExact(dimensionValues)); if (combinedValue != null) fieldValues.put(dimensionValues, combinedValue); } /** + * Makes a value unoverridable in a given context. + */ + public void setOverridable(String fieldName, boolean overridable, DimensionValues dimensionValues) { + getVariant(dimensionValues, true).setOverridable(fieldName, overridable); + } + + public Boolean isOverridable(String fieldName, DimensionValues dimensionValues) { + QueryProfileVariant variant = getVariant(dimensionValues, false); + if (variant == null) return null; + return variant.isOverridable(fieldName); + } + + /** * Returns the dimensions over which the virtual profiles in this may return different values. * Each dimension is a name for which a key-value may be supplied in the context properties * on lookup time to influence the value returned. @@ -257,6 +269,7 @@ public class QueryProfileVariants implements Freezable, Cloneable { return Collections.unmodifiableList(variants); } + @Override public QueryProfileVariants clone() { try { if (frozen) return this; @@ -304,9 +317,12 @@ public class QueryProfileVariants implements Freezable, Cloneable { return variant; } + @Override + public String toString() { return "variants of " + owner; } + public static class FieldValues implements Freezable, Cloneable { - private List<FieldValue> resolutionList=null; + private List<FieldValue> resolutionList = null; private boolean frozen = false; @@ -420,7 +436,7 @@ public class QueryProfileVariants implements Freezable, Cloneable { public static class FieldValue implements Comparable<FieldValue>, Cloneable { - private DimensionValues dimensionValues; + private final DimensionValues dimensionValues; private Object value; public FieldValue(DimensionValues dimensionValues, Object value) { @@ -458,7 +474,7 @@ public class QueryProfileVariants implements Freezable, Cloneable { } /** Clone by filling in the value from the given variants */ - public FieldValue clone(String fieldName,List<QueryProfileVariant> clonedVariants) { + public FieldValue clone(String fieldName, List<QueryProfileVariant> clonedVariants) { try { FieldValue clone = (FieldValue)super.clone(); if (this.value instanceof QueryProfile) @@ -471,6 +487,7 @@ public class QueryProfileVariants implements Freezable, Cloneable { } } + @Override public FieldValue clone() { try { FieldValue clone = (FieldValue)super.clone(); @@ -490,6 +507,9 @@ public class QueryProfileVariants implements Freezable, Cloneable { return null; } + @Override + public String toString() { return "field value " + value + " for " + dimensionValues; } + } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java index 2774bd4ebf2..46430a3041a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java @@ -33,8 +33,7 @@ public class Binding implements Comparable<Binding> { private final int hashCode; - @SuppressWarnings("unchecked") - public static final Binding nullBinding = new Binding(Integer.MAX_VALUE, Collections.<String,String>emptyMap()); + public static final Binding nullBinding = new Binding(Integer.MAX_VALUE, Map.of()); public static Binding createFrom(DimensionBinding dimensionBinding) { if (dimensionBinding.getDimensions().size() > maxDimensions) @@ -57,27 +56,65 @@ public class Binding implements Comparable<Binding> { return new Binding(generality, context); } - private Binding(int generality, Map<String, String> binding) { + /** Creates a binding from a map containing the exact bindings this will have */ + private Binding(int generality, Map<String, String> bindings) { this.generality = generality; // Map -> arrays to limit memory consumption and speed up evaluation - dimensions = new String[binding.size()]; - dimensionValues = new String[binding.size()]; + dimensions = new String[bindings.size()]; + dimensionValues = new String[bindings.size()]; int i = 0; - int bindingHash = 0; - for (Map.Entry<String,String> entry : binding.entrySet()) { + for (Map.Entry<String,String> entry : bindings.entrySet()) { dimensions[i] = entry.getKey(); dimensionValues[i] = entry.getValue(); - bindingHash += i * entry.getKey().hashCode() + 11 * i * entry.getValue().hashCode(); i++; } - this.hashCode = bindingHash; + this.hashCode = Arrays.hashCode(dimensions) + 11 * Arrays.hashCode(dimensionValues); + } + + Binding(DimensionalValue.BindingSpec spec, Map<String, String> bindings) { + this.generality = 0; // Not used here + + // Map -> arrays to limit memory consumption and speed up evaluation + dimensions = spec.dimensions(); + dimensionValues = new String[spec.dimensions().length]; + for (int i = 0; i < dimensions.length; i++) { + dimensionValues[i] = bindings.get(dimensions[i]); + } + this.hashCode = Arrays.hashCode(dimensions) + 11 * Arrays.hashCode(dimensionValues); + } + + /** + * Returns whether this binding is a proper generalization of the given binding: + * Meaning it contains a proper subset of the given bindings. + */ + public boolean generalizes(Binding other) { + if ( this.dimensions.length >= other.dimensions.length) return false; + for (int i = 0; i < this.dimensions.length; i++) { + int otherIndexOfDimension = this.indexOf(dimensions[i], other.dimensions); + if (otherIndexOfDimension < 0) return false; + if ( ! this.dimensionValues[i].equals(other.dimensionValues[otherIndexOfDimension])) return false; + } + return true; + } + + private int indexOf(String value, String[] array) { + for (int i = 0; i < array.length; i++) { + if (array[i].equals(value)) + return i; + } + return -1; } /** Returns true only if this binding is null (contains no values for its dimensions (if any) */ public boolean isNull() { return dimensions.length == 0; } + /** Do not change the returtned array */ + String[] dimensions() { return dimensions; } + + String[] dimensionValues() { return dimensionValues; } + @Override public String toString() { StringBuilder b = new StringBuilder("Binding["); @@ -106,7 +143,7 @@ public class Binding implements Comparable<Binding> { * Returns true if all the dimension values in this have the same values * in the given context. */ - public boolean matches(Map<String,String> context) { + public boolean matches(Map<String, String> context) { for (int i = 0; i < dimensions.length; i++) { if ( ! dimensionValues[i].equals(context.get(dimensions[i]))) return false; } @@ -116,7 +153,7 @@ public class Binding implements Comparable<Binding> { /** * Implements a partial ordering where more specific bindings come before less specific ones, * taking both the number of bindings and their positions into account (earlier dimensions - * take precedence over later ones. + * take precedence over later ones). * <p> * The order is not well defined for bindings in different dimensional spaces. */ diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java index 644d366e7d0..2439908183c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java @@ -30,26 +30,26 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable private final QueryProfileType type; /** The values of this */ - private final DimensionalMap<CompoundName, ValueWithSource> entries; + private final DimensionalMap<ValueWithSource> entries; /** Keys which have a type in this */ - private final DimensionalMap<CompoundName, QueryProfileType> types; + private final DimensionalMap<QueryProfileType> types; /** Keys which are (typed or untyped) references to other query profiles in this. Used as a set. */ - private final DimensionalMap<CompoundName, Object> references; + private final DimensionalMap<Object> references; /** Values which are not overridable in this. Used as a set. */ - private final DimensionalMap<CompoundName, Object> unoverridables; + private final DimensionalMap<Object> unoverridables; /** * Creates a new query profile from an id. */ public CompiledQueryProfile(ComponentId id, QueryProfileType type, - DimensionalMap<CompoundName, ValueWithSource> entries, - DimensionalMap<CompoundName, QueryProfileType> types, - DimensionalMap<CompoundName, Object> references, - DimensionalMap<CompoundName, Object> unoverridables, + DimensionalMap<ValueWithSource> entries, + DimensionalMap<QueryProfileType> types, + DimensionalMap<Object> references, + DimensionalMap<Object> unoverridables, CompiledQueryProfileRegistry registry) { super(id); this.registry = registry; @@ -91,10 +91,10 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable } /** Returns the types reachable from this, or an empty map (never null) if none */ - public DimensionalMap<CompoundName, QueryProfileType> getTypes() { return types; } + public DimensionalMap<QueryProfileType> getTypes() { return types; } /** Returns the references reachable from this, or an empty map (never null) if none */ - public DimensionalMap<CompoundName, Object> getReferences() { return references; } + public DimensionalMap<Object> getReferences() { return references; } /** * Return all objects that start with the given prefix path using no context. Use "" to list all. @@ -102,7 +102,7 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") * will return {"d" => "a.d-value","e" => "a.e-value"} */ - public final Map<String, Object> listValues(CompoundName prefix) { return listValues(prefix, Collections.<String,String>emptyMap()); } + public final Map<String, Object> listValues(CompoundName prefix) { return listValues(prefix, Collections.emptyMap()); } public final Map<String, Object> listValues(String prefix) { return listValues(new CompoundName(prefix)); } /** @@ -134,7 +134,6 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable public Map<String, Object> listValues(CompoundName prefix, Map<String, String> context, Properties substitution) { Map<String, Object> values = new HashMap<>(); for (Map.Entry<CompoundName, DimensionalValue<ValueWithSource>> entry : entries.entrySet()) { - if ( entry.getKey().size() <= prefix.size()) continue; if ( ! entry.getKey().hasPrefix(prefix)) continue; ValueWithSource valueWithSource = entry.getValue().get(context); @@ -160,6 +159,7 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable ValueWithSource valueWithSource = entry.getValue().get(context); if (valueWithSource == null) continue; + if (valueWithSource.value() == null) continue; valueWithSource = valueWithSource.withValue(substitute(valueWithSource.value(), context, substitution)); CompoundName suffixName = entry.getKey().rest(prefix.size()); @@ -183,6 +183,11 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable return substitute(value.value(), context, substitution); } + /** Returns all the entries from the profile **/ + public final DimensionalMap<ValueWithSource> getEntries() { + return this.entries; + } + private Object substitute(Object value, Map<String, String> context, Properties substitution) { if (value == null) return value; if (substitution == null) return value; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java index 7ab05d0cd1e..744c6eb6933 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java @@ -1,9 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.compiled; +import com.google.inject.Inject; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileCompiler; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; /** @@ -18,6 +23,15 @@ public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQuer private final QueryProfileTypeRegistry typeRegistry; + @Inject + public CompiledQueryProfileRegistry(QueryProfilesConfig config) { + QueryProfileRegistry registry = QueryProfileConfigurer.createFromConfig(config); + typeRegistry = registry.getTypeRegistry(); + for (QueryProfile inputProfile : registry.allComponents()) { + register(QueryProfileCompiler.compile(inputProfile, this)); + } + } + /** Creates a compiled query profile registry with no types */ public CompiledQueryProfileRegistry() { this(QueryProfileTypeRegistry.emptyFrozen()); @@ -28,7 +42,7 @@ public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQuer } /** Registers a type by its id */ - public void register(CompiledQueryProfile profile) { + public final void register(CompiledQueryProfile profile) { super.register(profile.getId(), profile); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java index 2e8f5dcf91c..6dc5f61c1f6 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java @@ -2,6 +2,7 @@ package com.yahoo.search.query.profile.compiled; import com.google.common.collect.ImmutableMap; +import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.profile.DimensionBinding; import java.util.HashMap; @@ -16,23 +17,23 @@ import java.util.Set; * * @author bratseth */ -public class DimensionalMap<KEY, VALUE> { +public class DimensionalMap<VALUE> { - private final Map<KEY, DimensionalValue<VALUE>> values; + private final Map<CompoundName, DimensionalValue<VALUE>> values; - private DimensionalMap(Map<KEY, DimensionalValue<VALUE>> values) { + private DimensionalMap(Map<CompoundName, DimensionalValue<VALUE>> values) { this.values = ImmutableMap.copyOf(values); } /** Returns the value for this key matching a context, or null if none */ - public VALUE get(KEY key, Map<String, String> context) { + public VALUE get(CompoundName key, Map<String, String> context) { DimensionalValue<VALUE> variants = values.get(key); if (variants == null) return null; return variants.get(context); } /** Returns the set of dimensional entries across all contexts. */ - public Set<Map.Entry<KEY, DimensionalValue<VALUE>>> entrySet() { + public Set<Map.Entry<CompoundName, DimensionalValue<VALUE>>> entrySet() { return values.entrySet(); } @@ -41,23 +42,18 @@ public class DimensionalMap<KEY, VALUE> { return values.isEmpty(); } - public static class Builder<KEY, VALUE> { + public static class Builder<VALUE> { - private Map<KEY, DimensionalValue.Builder<VALUE>> entries = new HashMap<>(); + private final Map<CompoundName, DimensionalValue.Builder<VALUE>> entries = new HashMap<>(); - // TODO: DimensionBinding -> Binding? - public void put(KEY key, DimensionBinding binding, VALUE value) { - DimensionalValue.Builder<VALUE> entry = entries.get(key); - if (entry == null) { - entry = new DimensionalValue.Builder<>(); - entries.put(key, entry); - } - entry.add(value, binding); + public void put(CompoundName key, Binding binding, VALUE value) { + entries.computeIfAbsent(key, __ -> new DimensionalValue.Builder<>()) + .add(value, binding); } - public DimensionalMap<KEY, VALUE> build() { - Map<KEY, DimensionalValue<VALUE>> map = new HashMap<>(); - for (Map.Entry<KEY, DimensionalValue.Builder<VALUE>> entry : entries.entrySet()) { + public DimensionalMap<VALUE> build() { + Map<CompoundName, DimensionalValue<VALUE>> map = new HashMap<>(); + for (Map.Entry<CompoundName, DimensionalValue.Builder<VALUE>> entry : entries.entrySet()) { map.put(entry.getKey(), entry.getValue().build(entries)); } return new DimensionalMap<>(map); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java index b5481059ac0..afe07b09d41 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java @@ -6,11 +6,13 @@ import com.yahoo.search.query.profile.DimensionBinding; import com.yahoo.search.query.profile.SubstituteString; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -20,20 +22,22 @@ import java.util.Set; */ public class DimensionalValue<VALUE> { - private final List<Value<VALUE>> values; + private final Map<Binding, VALUE> indexedVariants; + private final List<BindingSpec> bindingSpecs; - /** Create a set of variants which is a single value regardless of dimensions */ - public DimensionalValue(Value<VALUE> value) { - this.values = Collections.singletonList(value); - } + private DimensionalValue(List<Value<VALUE>> variants) { + Collections.sort(variants); - public DimensionalValue(List<Value<VALUE>> valueVariants) { - if (valueVariants.size() == 1) { // special cased for efficiency - this.values = Collections.singletonList(valueVariants.get(0)); - } - else { - this.values = new ArrayList<>(valueVariants); - Collections.sort(this.values); + // If there are inconsistent definitions of the same property, we should pick the first in the sort order + this.indexedVariants = new HashMap<>(); + for (Value<VALUE> variant : variants) + indexedVariants.putIfAbsent(variant.binding(), variant.value()); + + this.bindingSpecs = new ArrayList<>(); + for (Value<VALUE> variant : variants) { + BindingSpec spec = new BindingSpec(variant.binding()); + if ( ! bindingSpecs.contains(spec)) + bindingSpecs.add(spec); } } @@ -41,27 +45,30 @@ public class DimensionalValue<VALUE> { public VALUE get(Map<String, String> context) { if (context == null) context = Collections.emptyMap(); - for (Value<VALUE> value : values) { - if (value.matches(context)) - return value.value(); + + for (BindingSpec spec : bindingSpecs) { + if ( ! spec.matches(context)) continue; + VALUE value = indexedVariants.get(new Binding(spec, context)); + if (value != null) + return value; } return null; } - public boolean isEmpty() { return values.isEmpty(); } + public boolean isEmpty() { return indexedVariants.isEmpty(); } @Override public String toString() { - return values.toString(); + return indexedVariants.toString(); } public static class Builder<VALUE> { - /** The minimal set of variants needed to capture all values at this key */ - private Map<VALUE, Value.Builder<VALUE>> buildableVariants = new HashMap<>(); + /** The variants of the value of this key */ + private final Map<VALUE, Value.Builder<VALUE>> buildableVariants = new HashMap<>(); /** Returns the value for the given binding, or null if none */ - public VALUE valueFor(DimensionBinding variantBinding) { + public VALUE valueFor(Binding variantBinding) { for (var entry : buildableVariants.entrySet()) { if (entry.getValue().variants.contains(variantBinding)) return entry.getKey(); @@ -69,34 +76,36 @@ public class DimensionalValue<VALUE> { return null; } - public void add(VALUE value, DimensionBinding variantBinding) { + public void add(VALUE value, Binding variantBinding) { // Note: We know we can index by the value because its possible types are constrained - // to what query profiles allow: String, primitives and query profiles - Value.Builder variant = buildableVariants.get(value); - if (variant == null) { - variant = new Value.Builder<>(value); - buildableVariants.put(value, variant); - } - variant.addVariant(variantBinding); + // to what query profiles allow: String, primitives and query profiles (wrapped as a ValueWithSource) + buildableVariants.computeIfAbsent(value, Value.Builder::new) + .addVariant(variantBinding, value); } - public DimensionalValue<VALUE> build(Map<?, DimensionalValue.Builder<VALUE>> entries) { - List<Value> variants = new ArrayList<>(); - for (Value.Builder buildableVariant : buildableVariants.values()) { + public DimensionalValue<VALUE> build(Map<CompoundName, DimensionalValue.Builder<VALUE>> entries) { + List<Value<VALUE>> variants = new ArrayList<>(); + if (buildableVariants.size() == 1) { + // Compact size 1 as it is common and easy to do. To compact size > 1 we would need to + // compact within generic intervals having the same value + for (Value.Builder<VALUE> buildableVariant : buildableVariants.values()) + buildableVariant.compact(); + } + for (Value.Builder<VALUE> buildableVariant : buildableVariants.values()) { variants.addAll(buildableVariant.build(entries)); } - return new DimensionalValue(variants); + return new DimensionalValue<>(variants); } } /** A value for a particular binding */ - private static class Value<VALUE> implements Comparable<Value> { + private static class Value<VALUE> implements Comparable<Value<VALUE>> { - private VALUE value = null; + private final VALUE value; /** The minimal binding this holds for */ - private Binding binding = null; + private final Binding binding; public Value(VALUE value, Binding binding) { this.value = value; @@ -126,40 +135,78 @@ public class DimensionalValue<VALUE> { return " value '" + value + "' for " + binding; } - /** - * A single value with the minimal set of dimension combinations it holds for. - */ private static class Builder<VALUE> { - private final VALUE value; + private VALUE value; /** * The set of bindings this value is for. * Some of these are more general versions of others. * We need to keep both to allow interleaving a different value with medium generality. */ - private Set<DimensionBinding> variants = new HashSet<>(); + private List<Binding> variants = new ArrayList<>(); public Builder(VALUE value) { this.value = value; } /** Add a binding this holds for */ - public void addVariant(DimensionBinding binding) { + @SuppressWarnings("unchecked") + public void addVariant(Binding binding, VALUE newValue) { variants.add(binding); + + // We're combining values for efficiency, so remove incorrect provenance info + if (value instanceof ValueWithSource) { + ValueWithSource v1 = (ValueWithSource)value; + ValueWithSource v2 = (ValueWithSource)newValue; + + if (v1.source() != null && ! v1.source().equals(v2.source())) + v1 = v1.withSource(null); + + // We could keep the more general variant here (when matching), but that situation is rare + if (v1.variant().isPresent() && ! v1.variant().equals(v2.variant())) + v1 = v1.withVariant(Optional.empty()); + + value = (VALUE)v1; + } + } + + /** Remove variants that are specializations of other variants in this */ + void compact() { + Collections.sort(variants); + List<Binding> compacted = new ArrayList<>(); + + if (variants.get(variants.size() - 1).dimensions().length == 0) { // Shortcut + variants = List.of(variants.get(variants.size() - 1)); + } + else { + for (int i = variants.size() - 1; i >= 0; i--) { + if ( ! containsGeneralizationOf(variants.get(i), compacted)) + compacted.add(variants.get(i)); + } + Collections.reverse(compacted); + variants = compacted; + } + } + + private boolean containsGeneralizationOf(Binding binding, List<Binding> bindings) { + for (Binding candidate : bindings) { + if (candidate.generalizes(binding)) + return true; + } + return false; } /** Build a separate value object for each dimension combination which has this value */ public List<Value<VALUE>> build(Map<CompoundName, DimensionalValue.Builder<VALUE>> entries) { - // Shortcut for efficiency of the normal case - if (variants.size() == 1) { - return Collections.singletonList(new Value<>(substituteIfRelative(value, variants.iterator().next(), entries), - Binding.createFrom(variants.iterator().next()))); + if (variants.size() == 1) { // Shortcut + return List.of(new Value<>(substituteIfRelative(value, variants.iterator().next(), entries), + variants.iterator().next())); } List<Value<VALUE>> values = new ArrayList<>(variants.size()); - for (DimensionBinding variant : variants) { - values.add(new Value<>(substituteIfRelative(value, variant, entries), Binding.createFrom(variant))); + for (Binding variant : variants) { + values.add(new Value<>(substituteIfRelative(value, variant, entries), variant)); } return values; } @@ -171,7 +218,7 @@ public class DimensionalValue<VALUE> { // TODO: Move this @SuppressWarnings("unchecked") private VALUE substituteIfRelative(VALUE value, - DimensionBinding variant, + Binding variant, Map<CompoundName, DimensionalValue.Builder<VALUE>> entries) { if (value instanceof ValueWithSource && ((ValueWithSource)value).value() instanceof SubstituteString) { ValueWithSource valueWithSource = (ValueWithSource)value; @@ -185,7 +232,7 @@ public class DimensionalValue<VALUE> { if (substituteValues == null) throw new IllegalArgumentException("Could not resolve local substitution '" + relativeComponent.fieldName() + "' in variant " + - variant); + Arrays.toString(variant.dimensionValues())); ValueWithSource resolved = (ValueWithSource)substituteValues.valueFor(variant); resolvedComponents.add(new SubstituteString.StringComponent(resolved.value().toString())); } @@ -212,4 +259,38 @@ public class DimensionalValue<VALUE> { } + /** A list of dimensions for which there exist one or more bindings in this */ + static class BindingSpec { + + /** The dimensions of this. Unenforced invariant: Content never changes. */ + private final String[] dimensions; + + public BindingSpec(Binding binding) { + this.dimensions = binding.dimensions(); + } + + /** Do not change the returned array */ + String[] dimensions() { return dimensions; } + + /** Returns whether this context contains all the keys of this */ + public boolean matches(Map<String, String> context) { + for (int i = 0; i < dimensions.length; i++) + if ( ! context.containsKey(dimensions[i])) return false; + return true; + } + + @Override + public int hashCode() { + return Arrays.hashCode(dimensions); + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof BindingSpec)) return false; + return Arrays.equals(((BindingSpec)other).dimensions, this.dimensions); + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/ValueWithSource.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/ValueWithSource.java index 925d20903c6..53a29b74813 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/ValueWithSource.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/ValueWithSource.java @@ -2,7 +2,9 @@ package com.yahoo.search.query.profile.compiled; import com.yahoo.search.query.profile.DimensionValues; +import com.yahoo.search.query.profile.types.QueryProfileType; +import java.util.Objects; import java.util.Optional; /** @@ -14,35 +16,91 @@ public class ValueWithSource { private final Object value; - /** The source of the query profile having a value */ private final String source; + private final boolean isUnoverridable; + + private final boolean isQueryProfile; + + private final QueryProfileType type; + /** The dimension values specifying a variant in that profile, or null if it is not in a variant */ private final DimensionValues variant; - public ValueWithSource(Object value, String source, DimensionValues variant) { + public ValueWithSource(Object value, + String source, + boolean isUnoverridable, boolean isQueryProfile, QueryProfileType type, + DimensionValues variant) { this.value = value; this.source = source; + this.isUnoverridable = isUnoverridable; + this.isQueryProfile = isQueryProfile; + this.type = type; this.variant = variant; } + /** + * Returns the value at this key, or null if none + * (in which case this is references a query profile which has no value set). + */ public Object value() { return value; } + /** Returns the source of the query profile having a value */ public String source() { return source; } + /** Returns true if this value cannot be overridden in queries */ + public boolean isUnoverridable() { return isUnoverridable; } + + /** + * Returns true if this key references a query profile (i.e a non-leaf). + * In this case the value may or may not be null, as non-leafs may have values. + */ + public boolean isQueryProfile() { return isQueryProfile; } + + /** Returns tye type of this if it refers to a query profile (not a leaf value), and it has a type */ + public QueryProfileType queryProfileType() { return type; } + public ValueWithSource withValue(Object value) { - return new ValueWithSource(value, source, variant); + return new ValueWithSource(value, source, isUnoverridable, isQueryProfile, type, variant); + } + + public ValueWithSource withSource(String source) { + return new ValueWithSource(value, source, isUnoverridable, isQueryProfile, type, variant); + } + + public ValueWithSource withVariant(Optional<DimensionValues> variant) { + return new ValueWithSource(value, source, isUnoverridable, isQueryProfile, type, variant.orElse(null)); } /** Returns the variant having this value, or empty if it's not in a variant */ public Optional<DimensionValues> variant() { return Optional.ofNullable(variant); } @Override + public int hashCode() { + // Value is always a value object. Don't include source in identity. + return Objects.hash(value, isUnoverridable, type); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof ValueWithSource)) return false; + + ValueWithSource other = (ValueWithSource)o; + if ( ! Objects.equals(this.value, other.value)) return false; + if ( ! Objects.equals(this.isUnoverridable, other.isUnoverridable)) return false; + if ( ! Objects.equals(this.type, other.type)) return false; + return true; + } + + @Override public String toString() { - return value + - " (from query profile '" + source + "'" + + if (source == null && variant == null) return value.toString(); + + return value + " (" + + ( source != null ? "from query profile '" + source + "'" : "") + ( variant != null ? " variant " + variant : "") + - ")"; + ")"; } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java index d3c232f84c5..423348454d0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java @@ -121,16 +121,20 @@ public class QueryProfileConfigurer implements ConfigSubscriber.SingleSubscriber QueryProfile referenced = registry.getComponent(referenceConfig.value()); if (referenced == null) throw new IllegalArgumentException("Query profile '" + referenceConfig.value() + "' referenced as '" + - referenceConfig.name() + "' in " + profile + " was not found"); + referenceConfig.name() + "' in " + profile + " was not found"); profile.set(referenceConfig.name(), referenced, registry); if (referenceConfig.overridable() != null && !referenceConfig.overridable().isEmpty()) - profile.setOverridable(referenceConfig.name(), BooleanParser.parseBoolean(referenceConfig.overridable()), null); + profile.setOverridable(referenceConfig.name(), + BooleanParser.parseBoolean(referenceConfig.overridable()), + DimensionValues.empty); } for (QueryProfilesConfig.Queryprofile.Property propertyConfig : config.property()) { profile.set(propertyConfig.name(), propertyConfig.value(), registry); if (propertyConfig.overridable() != null && ! propertyConfig.overridable().isEmpty()) - profile.setOverridable(propertyConfig.name(), BooleanParser.parseBoolean(propertyConfig.overridable()), null); + profile.setOverridable(propertyConfig.name(), + BooleanParser.parseBoolean(propertyConfig.overridable()), + DimensionValues.empty); } for (QueryProfilesConfig.Queryprofile.Queryprofilevariant variantConfig : config.queryprofilevariant()) { @@ -152,15 +156,24 @@ public class QueryProfileConfigurer implements ConfigSubscriber.SingleSubscriber } for (QueryProfilesConfig.Queryprofile.Queryprofilevariant.Reference referenceConfig : variantConfig.reference()) { - QueryProfile referenced=registry.getComponent(referenceConfig.value()); + QueryProfile referenced = registry.getComponent(referenceConfig.value()); if (referenced == null) throw new IllegalArgumentException("Query profile '" + referenceConfig.value() + "' referenced as '" + - referenceConfig.name() + "' in " + profile + " for '" + forDimensionValues + "' was not found"); + referenceConfig.name() + "' in " + profile + + " for '" + forDimensionValues + "' was not found"); profile.set(referenceConfig.name(), referenced, forDimensionValues, registry); + if ( ! referenceConfig.overridable().isEmpty()) + profile.setOverridable(referenceConfig.name(), + Boolean.parseBoolean(referenceConfig.overridable()), + forDimensionValues); } for (QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property propertyConfig : variantConfig.property()) { profile.set(propertyConfig.name(), propertyConfig.value(), forDimensionValues, registry); + if ( ! propertyConfig.overridable().isEmpty()) + profile.setOverridable(propertyConfig.name(), + Boolean.parseBoolean(propertyConfig.overridable()), + forDimensionValues); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java index 1b1cdce5890..a64f2087d1f 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java @@ -41,21 +41,21 @@ public class QueryProfileXMLReader { try { File dir = new File(directory); if ( ! dir.isDirectory() ) throw new IllegalArgumentException("Could not read query profiles: '" + - directory + "' is not a valid directory."); + directory + "' is not a valid directory."); for (File file : sortFiles(dir)) { if ( ! file.getName().endsWith(".xml")) continue; - queryProfileReaders.add(new NamedReader(file.getName(),new FileReader(file))); + queryProfileReaders.add(new NamedReader(file.getName(), new FileReader(file))); } File typeDir=new File(dir,"types"); if (typeDir.isDirectory()) { for (File file : sortFiles(typeDir)) { if ( ! file.getName().endsWith(".xml")) continue; - queryProfileTypeReaders.add(new NamedReader(file.getName(),new FileReader(file))); + queryProfileTypeReaders.add(new NamedReader(file.getName(), new FileReader(file))); } } - return read(queryProfileTypeReaders,queryProfileReaders); + return read(queryProfileTypeReaders, queryProfileReaders); } catch (IOException e) { throw new IllegalArgumentException("Could not read query profiles from '" + directory + "'", e); @@ -235,7 +235,7 @@ public class QueryProfileXMLReader { QueryProfile profile = registry.getComponent(new ComponentSpecification(element.getAttribute("id")).toId()); try { readInherited(element, profile, registry,null, profile.toString()); - readFields(element, profile, registry,null, profile.toString()); + readFields(element, profile, registry,DimensionValues.empty, profile.toString()); readVariants(element, profile, registry); } catch (RuntimeException e) { @@ -268,10 +268,9 @@ public class QueryProfileXMLReader { if (name == null || name.equals("")) throw new IllegalArgumentException("A field in " + sourceDescription + " has no 'name' attribute"); try { - Boolean overridable = getBooleanAttribute("overridable",null,field); + Boolean overridable = getBooleanAttribute("overridable", null, field); if (overridable != null) - profile.setOverridable(name, overridable, null); - + profile.setOverridable(name, overridable, dimensionValues.asContext(profile.getDimensions())); Object fieldValue = readFieldValue(field, name, sourceDescription, registry); if (fieldValue instanceof QueryProfile) references.add(new KeyValue(name, fieldValue)); @@ -358,8 +357,8 @@ public class QueryProfileXMLReader { private static class KeyValue { - private String key; - private Object value; + private final String key; + private final Object value; public KeyValue(String key, Object value) { this.key = key; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java new file mode 100644 index 00000000000..e5b9eb1c1cd --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types; + +import com.yahoo.language.Language; +import com.yahoo.language.process.Embedder; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; + +import java.util.Map; + +/** + * @author bratseth + */ +public class ConversionContext { + + private final CompiledQueryProfileRegistry registry; + private final Embedder embedder; + private final Language language; + + public ConversionContext(CompiledQueryProfileRegistry registry, Embedder embedder, Map<String, String> context) { + this.registry = registry; + this.embedder = embedder; + this.language = context.containsKey("language") ? Language.fromLanguageTag(context.get("language")) + : Language.UNKNOWN; + } + + /** Returns the profile registry, or null if none */ + CompiledQueryProfileRegistry getRegistry() {return registry;} + + /** Returns the configured encoder, never null */ + Embedder getEncoder() { return embedder; } + + /** Returns the language, which is never null but may be UNKNOWN */ + Language getLanguage() { return language; } + + /** Returns an empty context */ + public static ConversionContext empty() { + return new ConversionContext(null, Embedder.throwsOnUse, Map.of()); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java index 6c30f1a8b05..7f8836ef2c1 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java @@ -33,7 +33,7 @@ public class FieldDescription implements Comparable<FieldDescription> { } public FieldDescription(String name, String type) { - this(name,FieldType.fromString(type,null)); + this(name,FieldType.fromString(type, null)); } public FieldDescription(String name, FieldType type, boolean mandatory) { @@ -60,7 +60,7 @@ public class FieldDescription implements Comparable<FieldDescription> { * @param overridable whether this can be overridden when first set in a profile. Default: true */ public FieldDescription(String name, String typeString, String aliases, boolean mandatory, boolean overridable) { - this(name,FieldType.fromString(typeString,null),aliases,mandatory,overridable); + this(name,FieldType.fromString(typeString, null), aliases, mandatory, overridable); } public FieldDescription(String name, FieldType type, boolean mandatory, boolean overridable) { @@ -97,7 +97,8 @@ public class FieldDescription implements Comparable<FieldDescription> { this.type = type; // Forbidden until we can figure out the right semantics - if (name.isCompound() && ! aliases.isEmpty()) throw new IllegalArgumentException("Aliases are not allowed with compound names"); + if (name.isCompound() && ! aliases.isEmpty()) + throw new IllegalArgumentException("Aliases are not allowed with compound names"); this.aliases = ImmutableList.copyOf(aliases); this.mandatory = mandatory; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java index 3bfd33668e6..7a06f9ef534 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java @@ -3,7 +3,6 @@ package com.yahoo.search.query.profile.types; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.yql.YqlQuery; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -41,7 +40,7 @@ public abstract class FieldType { public abstract Object convertFrom(Object o, QueryProfileRegistry registry); /** Converts the given type to an instance of this type, if possible. Returns null if not possible. */ - public abstract Object convertFrom(Object o, CompiledQueryProfileRegistry registry); + public abstract Object convertFrom(Object o, ConversionContext context); /** * Returns this type as a tensor type: The true tensor type is this is a tensor field an an empty type - @@ -77,7 +76,7 @@ public abstract class FieldType { if ("query-profile".equals(typeString)) return genericQueryProfileType; if (typeString.startsWith("query-profile:")) - return QueryProfileFieldType.fromString(typeString.substring("query-profile:".length()),registry); + return QueryProfileFieldType.fromString(typeString.substring("query-profile:".length()), registry); throw new IllegalArgumentException("Unknown type '" + typeString + "'"); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java index 1e904e4f970..f9d8950908b 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java @@ -2,7 +2,6 @@ package com.yahoo.search.query.profile.types; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import static com.yahoo.text.Lowercase.toLowerCase; @@ -37,7 +36,7 @@ public class PrimitiveFieldType extends FieldType { } @Override - public Object convertFrom(Object object, CompiledQueryProfileRegistry registry) { + public Object convertFrom(Object object, ConversionContext context) { return convertFrom(object, (QueryProfileRegistry)null); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java index 1797a2bd59f..cbae6402039 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java @@ -2,7 +2,6 @@ package com.yahoo.search.query.profile.types; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.yql.YqlQuery; /** @@ -32,7 +31,7 @@ public class QueryFieldType extends FieldType { } @Override - public Object convertFrom(Object o, CompiledQueryProfileRegistry registry) { + public Object convertFrom(Object o, ConversionContext context) { return convertFrom(o, (QueryProfileRegistry)null); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java index fda2d27e682..ff12224823f 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java @@ -4,7 +4,6 @@ package com.yahoo.search.query.profile.types; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; /** * Represents a query profile field type which is a reference to a query profile. @@ -57,11 +56,11 @@ public class QueryProfileFieldType extends FieldType { } @Override - public CompiledQueryProfile convertFrom(Object object, CompiledQueryProfileRegistry registry) { + public CompiledQueryProfile convertFrom(Object object, ConversionContext context) { String profileId = object.toString(); if (profileId.startsWith("ref:")) profileId = profileId.substring("ref:".length()); - CompiledQueryProfile profile = registry.getComponent(profileId); + CompiledQueryProfile profile = context.getRegistry().getComponent(profileId); if (profile == null) return null; if (type != null && ! type.equals(profile.getType())) return null; return profile; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java index c02aada2062..e4396894595 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java @@ -177,11 +177,12 @@ public class QueryProfileType extends FreezableSimpleComponent { public void freeze() { if (isFrozen()) return; - // Flatten the inheritance hierarchy into this to facilitate faster lookup + // Flatten for faster lookup for (QueryProfileType inheritedType : inherited) { for (FieldDescription field : inheritedType.fields().values()) - if ( ! fields.containsKey(field.getName())) - fields.put(field.getName(),field); + if ( ! fields.containsKey(field.getName())) { + fields.put(field.getName(), field); + } } fields = ImmutableMap.copyOf(fields); inherited = ImmutableList.copyOf(inherited); @@ -354,9 +355,10 @@ public class QueryProfileType extends FreezableSimpleComponent { if (inherited().size() == 0) return Collections.unmodifiableMap(fields); // Collapse inherited - Map<String, FieldDescription> allFields = new HashMap<>(fields); + Map<String, FieldDescription> allFields = new HashMap<>(); for (QueryProfileType inheritedType : inherited) allFields.putAll(inheritedType.fields()); + allFields.putAll(fields); return Collections.unmodifiableMap(allFields); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java index 9699a72cb31..cd21f0b3a61 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java @@ -1,8 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; +import com.yahoo.language.Language; +import com.yahoo.language.process.Embedder; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -38,14 +39,26 @@ public class TensorFieldType extends FieldType { @Override public Object convertFrom(Object o, QueryProfileRegistry registry) { + return convertFrom(o, ConversionContext.empty()); + } + + @Override + public Object convertFrom(Object o, ConversionContext context) { + return convertFrom(o, context.getEncoder(), context.getLanguage()); + } + + private Object convertFrom(Object o, Embedder embedder, Language language) { if (o instanceof Tensor) return o; + if (o instanceof String && ((String)o).startsWith("embed(")) return encode((String)o, embedder, language); if (o instanceof String) return Tensor.from(type, (String)o); return null; } - @Override - public Object convertFrom(Object o, CompiledQueryProfileRegistry registry) { - return convertFrom(o, (QueryProfileRegistry)null); + private Tensor encode(String s, Embedder embedder, Language language) { + if ( ! s.endsWith(")")) + throw new IllegalArgumentException("Expected any string enclosed in embed(), but the argument does not end by ')'"); + String text = s.substring("embed(".length(), s.length() - 1); + return embedder.embed(text, language, type); } public static TensorFieldType fromTypeString(String s) { diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java index a4a82d27f8e..83e8dd530ad 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java @@ -37,6 +37,8 @@ public class PropertyAliases extends Properties { * @return the real name if an alias or the input name itself */ protected CompoundName unalias(CompoundName nameOrAlias) { + if (aliases.isEmpty()) return nameOrAlias; + if (nameOrAlias.size() > 1) return nameOrAlias; // aliases are simple names CompoundName properName = aliases.get(nameOrAlias.getLowerCasedName()); return (properName != null) ? properName : nameOrAlias; } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java index 4f30331e738..643e215daef 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java @@ -26,7 +26,10 @@ public class PropertyMap extends Properties { /** The properties of this */ private Map<CompoundName, Object> properties = new LinkedHashMap<>(); - public void set(CompoundName name, Object value, Map<String,String> context) { + public void set(CompoundName name, Object value, Map<String, String> context) { + if (value == null) // Both clear and forward + properties.remove(name); + if (shouldSet(name, value)) properties.put(name, value); else diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index dfe6c2af44b..3a426656185 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.properties; +import com.yahoo.language.process.Embedder; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; @@ -10,6 +12,7 @@ import com.yahoo.search.query.Properties; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.Select; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.ConversionContext; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.ranking.Diversity; @@ -31,10 +34,12 @@ public class QueryProperties extends Properties { private Query query; private final CompiledQueryProfileRegistry profileRegistry; + private final Embedder embedder; - public QueryProperties(Query query, CompiledQueryProfileRegistry profileRegistry) { + public QueryProperties(Query query, CompiledQueryProfileRegistry profileRegistry, Embedder embedder) { this.query = query; this.profileRegistry = profileRegistry; + this.embedder = embedder; } public void setParentQuery(Query query) { @@ -67,9 +72,10 @@ public class QueryProperties extends Properties { if (key.last().equals(Ranking.SORTING)) return ranking.getSorting(); if (key.last().equals(Ranking.FRESHNESS)) return ranking.getFreshness(); if (key.last().equals(Ranking.QUERYCACHE)) return ranking.getQueryCache(); + if (key.last().equals(Ranking.RERANKCOUNT)) return ranking.getRerankCount(); if (key.last().equals(Ranking.LIST_FEATURES)) return ranking.getListFeatures(); } - else if (key.size()>=3 && key.get(1).equals(Ranking.MATCH_PHASE)) { + else if (key.size() >= 3 && key.get(1).equals(Ranking.MATCH_PHASE)) { if (key.size() == 3) { MatchPhase matchPhase = ranking.getMatchPhase(); if (key.last().equals(MatchPhase.ATTRIBUTE)) return matchPhase.getAttribute(); @@ -144,7 +150,6 @@ public class QueryProperties extends Properties { return super.get(key, context, substitution); } - @SuppressWarnings("deprecation") @Override public void set(CompoundName key, Object value, Map<String,String> context) { // Note: The defaults here are never used @@ -172,7 +177,7 @@ public class QueryProperties extends Properties { else if (key.last().equals(Model.RESTRICT)) model.setRestrict(asString(value,"")); else - throwIllegalParameter(key.last(),Model.MODEL); + throwIllegalParameter(key.last(), Model.MODEL); } else if (key.first().equals(Ranking.RANKING)) { Ranking ranking = query.getRanking(); @@ -187,57 +192,85 @@ public class QueryProperties extends Properties { ranking.setFreshness(asString(value, "")); else if (key.last().equals(Ranking.QUERYCACHE)) ranking.setQueryCache(asBoolean(value, false)); + else if (key.last().equals(Ranking.RERANKCOUNT)) + ranking.setRerankCount(asInteger(value, null)); else if (key.last().equals(Ranking.LIST_FEATURES)) ranking.setListFeatures(asBoolean(value,false)); + else + throwIllegalParameter(key.last(), Ranking.RANKING); } - else if (key.size()>=3 && key.get(1).equals(Ranking.MATCH_PHASE)) { + else if (key.size() >= 3 && key.get(1).equals(Ranking.MATCH_PHASE)) { if (key.size() == 3) { MatchPhase matchPhase = ranking.getMatchPhase(); - if (key.last().equals(MatchPhase.ATTRIBUTE)) { + if (key.last().equals(MatchPhase.ATTRIBUTE)) matchPhase.setAttribute(asString(value, null)); - } else if (key.last().equals(MatchPhase.ASCENDING)) { + else if (key.last().equals(MatchPhase.ASCENDING)) matchPhase.setAscending(asBoolean(value, false)); - } else if (key.last().equals(MatchPhase.MAX_HITS)) { + else if (key.last().equals(MatchPhase.MAX_HITS)) matchPhase.setMaxHits(asLong(value, null)); - } else if (key.last().equals(MatchPhase.MAX_FILTER_COVERAGE)) { + else if (key.last().equals(MatchPhase.MAX_FILTER_COVERAGE)) matchPhase.setMaxFilterCoverage(asDouble(value, 0.2)); - } + else + throwIllegalParameter(key.rest().toString(), Ranking.MATCH_PHASE); } else if (key.size() > 3 && key.get(2).equals(Ranking.DIVERSITY)) { Diversity diversity = ranking.getMatchPhase().getDiversity(); if (key.last().equals(Diversity.ATTRIBUTE)) { diversity.setAttribute(asString(value, null)); - } else if (key.last().equals(Diversity.MINGROUPS)) { + } + else if (key.last().equals(Diversity.MINGROUPS)) { diversity.setMinGroups(asLong(value, null)); - } else if ((key.size() > 4) && key.get(3).equals(Diversity.CUTOFF)) { - if (key.last().equals(Diversity.FACTOR)) { + } + else if ((key.size() > 4) && key.get(3).equals(Diversity.CUTOFF)) { + if (key.last().equals(Diversity.FACTOR)) diversity.setCutoffFactor(asDouble(value, 10.0)); - } else if (key.last().equals(Diversity.STRATEGY)) { + else if (key.last().equals(Diversity.STRATEGY)) diversity.setCutoffStrategy(asString(value, "loose")); - } + else + throwIllegalParameter(key.rest().toString(), Diversity.CUTOFF); + } + else { + throwIllegalParameter(key.rest().toString(), Ranking.DIVERSITY); } } } else if (key.size() == 3 && key.get(1).equals(Ranking.SOFTTIMEOUT)) { SoftTimeout soft = ranking.getSoftTimeout(); - if (key.last().equals(SoftTimeout.ENABLE)) soft.setEnable(asBoolean(value, true)); - if (key.last().equals(SoftTimeout.FACTOR)) soft.setFactor(asDouble(value, null)); - if (key.last().equals(SoftTimeout.TAILCOST)) soft.setTailcost(asDouble(value, null)); + if (key.last().equals(SoftTimeout.ENABLE)) + soft.setEnable(asBoolean(value, true)); + else if (key.last().equals(SoftTimeout.FACTOR)) + soft.setFactor(asDouble(value, null)); + else if (key.last().equals(SoftTimeout.TAILCOST)) + soft.setTailcost(asDouble(value, null)); + else + throwIllegalParameter(key.rest().toString(), Ranking.SOFTTIMEOUT); } else if (key.size() == 3 && key.get(1).equals(Ranking.MATCHING)) { Matching matching = ranking.getMatching(); - if (key.last().equals(Matching.TERMWISELIMIT)) matching.setTermwiselimit(asDouble(value, 1.0)); - if (key.last().equals(Matching.NUMTHREADSPERSEARCH)) matching.setNumThreadsPerSearch(asInteger(value, 1)); - if (key.last().equals(Matching.NUMSEARCHPARTITIIONS)) matching.setNumSearchPartitions(asInteger(value, 1)); - if (key.last().equals(Matching.MINHITSPERTHREAD)) matching.setMinHitsPerThread(asInteger(value, 0)); + if (key.last().equals(Matching.TERMWISELIMIT)) + matching.setTermwiselimit(asDouble(value, 1.0)); + else if (key.last().equals(Matching.NUMTHREADSPERSEARCH)) + matching.setNumThreadsPerSearch(asInteger(value, 1)); + else if (key.last().equals(Matching.NUMSEARCHPARTITIIONS)) + matching.setNumSearchPartitions(asInteger(value, 1)); + else if (key.last().equals(Matching.MINHITSPERTHREAD)) + matching.setMinHitsPerThread(asInteger(value, 0)); + else + throwIllegalParameter(key.rest().toString(), Ranking.MATCHING); } else if (key.size() > 2) { String restKey = key.rest().rest().toString(); if (key.get(1).equals(Ranking.FEATURES)) - setRankingFeature(query, restKey, toSpecifiedType(restKey, value, profileRegistry.getTypeRegistry().getComponent("features"))); + setRankingFeature(query, restKey, toSpecifiedType(restKey, + value, + profileRegistry.getTypeRegistry().getComponent("features"), + context)); else if (key.get(1).equals(Ranking.PROPERTIES)) - ranking.getProperties().put(restKey, toSpecifiedType(restKey, value, profileRegistry.getTypeRegistry().getComponent("properties"))); + ranking.getProperties().put(restKey, toSpecifiedType(restKey, + value, + profileRegistry.getTypeRegistry().getComponent("properties"), + context)); else - throwIllegalParameter(key.rest().toString(),Ranking.RANKING); + throwIllegalParameter(key.rest().toString(), Ranking.RANKING); } } else if (key.size() == 2 && key.first().equals(Presentation.PRESENTATION)) { @@ -259,20 +292,27 @@ public class QueryProperties extends Properties { query.getSelect().setGroupingExpressionString(asString(value, "")); } else if (key.size() == 2) { - if (key.last().equals(Select.WHERE)) { + if (key.last().equals(Select.WHERE)) query.getSelect().setWhereString(asString(value, "")); - } else if (key.last().equals(Select.GROUPING)) { + else if (key.last().equals(Select.GROUPING)) query.getSelect().setGroupingString(asString(value, "")); - } + else + throwIllegalParameter(key.rest().toString(), Select.SELECT); } else { throwIllegalParameter(key.last(), Select.SELECT); } } else if (key.first().equals("rankfeature") || key.first().equals("featureoverride") ) { // featureoverride is deprecated - setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("features"))); + setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(), + value, + profileRegistry.getTypeRegistry().getComponent("features"), + context)); } else if (key.first().equals("rankproperty")) { - query.getRanking().getProperties().put(key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("properties"))); + query.getRanking().getProperties().put(key.rest().toString(), toSpecifiedType(key.rest().toString(), + value, + profileRegistry.getTypeRegistry().getComponent("properties"), + context)); } else if (key.size()==1) { if (key.equals(Query.HITS)) query.setHits(asInteger(value,10)); @@ -294,10 +334,10 @@ public class QueryProperties extends Properties { super.set(key,value,context); } catch (Exception e) { // Make sure error messages are informative. This should be moved out of this properties implementation - if (e.getMessage().startsWith("Could not set")) + if (e.getMessage() != null && e.getMessage().startsWith("Could not set")) throw e; else - throw new IllegalArgumentException("Could not set '" + key + "' to '" + value + "'", e); + throw new IllegalInputException("Could not set '" + key + "' to '" + value + "'", e); } } @@ -335,17 +375,17 @@ public class QueryProperties extends Properties { } } - private Object toSpecifiedType(String key, Object value, QueryProfileType type) { + private Object toSpecifiedType(String key, Object value, QueryProfileType type, Map<String,String> context) { if ( ! ( value instanceof String)) return value; // already typed if (type == null) return value; // no type info -> keep as string FieldDescription field = type.getField(key); if (field == null) return value; // ditto - return field.getType().convertFrom(value, profileRegistry); + return field.getType().convertFrom(value, new ConversionContext(profileRegistry, embedder, context)); } private void throwIllegalParameter(String key,String namespace) { - throw new IllegalArgumentException("'" + key + "' is not a valid property in '" + namespace + - "'. See the search api for valid keys starting by '" + namespace + "'."); + throw new IllegalInputException("'" + key + "' is not a valid property in '" + namespace + + "'. See the query api for valid keys starting by '" + namespace + "'."); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java b/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java index 794247863bf..72a6533e946 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.ranking; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.profile.types.FieldDescription; @@ -84,8 +85,9 @@ public class MatchPhase implements Cloneable { public void setMaxFilterCoverage(double maxFilterCoverage) { if ((maxFilterCoverage < 0.0) || (maxFilterCoverage > 1.0)) { - throw new IllegalArgumentException("maxFilterCoverage must be in the range [0.0, 1.0]. It is " + maxFilterCoverage); + throw new IllegalInputException("maxFilterCoverage must be in the range [0.0, 1.0]. It is " + maxFilterCoverage); } + this.maxFilterCoverage = maxFilterCoverage; } diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java b/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java index fb3f2acfadd..14e4e006b39 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.ranking; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; @@ -59,7 +60,7 @@ public class Matching implements Cloneable { public void setTermwiselimit(double value) { if ((value < 0.0) || (value > 1.0)) { - throw new IllegalArgumentException("termwiselimit must be in the range [0.0, 1.0]. It is " + value); + throw new IllegalInputException("termwiselimit must be in the range [0.0, 1.0]. It is " + value); } termwiseLimit = value; } diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java index 0d47ef77ce5..43c26692221 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.ranking; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.profile.types.FieldDescription; @@ -54,7 +55,7 @@ public class SoftTimeout implements Cloneable { /** Override the adaptive factor determined on the content nodes */ public void setFactor(double factor) { if ((factor < 0.0) || (factor > 1.0)) { - throw new IllegalArgumentException("factor must be in the range [0.0, 1.0], got " + factor); + throw new IllegalInputException("factor must be in the range [0.0, 1.0], got " + factor); } this.factor = factor; } @@ -64,7 +65,7 @@ public class SoftTimeout implements Cloneable { /** Override the tail cost factor determined on the content nodes */ public void setTailcost(double tailcost) { if ((tailcost < 0.0) || (tailcost > 1.0)) { - throw new IllegalArgumentException("tailcost must be in the range [0.0, 1.0], got " + tailcost); + throw new IllegalInputException("tailcost must be in the range [0.0, 1.0], got " + tailcost); } this.tailcost = tailcost; } diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java index b73ef1298ee..af73e905f20 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java @@ -2,7 +2,7 @@ package com.yahoo.search.query.rewrite; import com.yahoo.fsa.FSA; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.search.Query; import com.yahoo.search.intent.model.IntentModel; import com.yahoo.search.intent.model.InterpretationNode; @@ -289,7 +289,7 @@ public class RewriterUtils { * @param msg Log message */ public static void log(Logger logger, String msg) { - logger.log(LogLevel.DEBUG, logger.getName() + ": " + msg); + logger.log(Level.FINE, () -> logger.getName() + ": " + msg); } /** @@ -303,7 +303,7 @@ public class RewriterUtils { if(query!=null) { query.trace(logger.getName() + ": " + msg, true, TRACELEVEL); } - logger.log(LogLevel.DEBUG, logger.getName() + ": " + msg); + logger.log(Level.FINE, () -> logger.getName() + ": " + msg); } /** diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java index 794217e688f..9a2374da4c9 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java @@ -76,9 +76,8 @@ public class GenericExpansionRewriter extends QueryRewriteSearcher { HashMap<String, File> fileList) { logger = Logger.getLogger(GenericExpansionRewriter.class.getName()); FSA fsa = (FSA)rewriterDicts.get(GENERIC_EXPAND_DICT); - if(fsa==null) { - RewriterUtils.error(logger, "Error retrieving FSA dictionary: " + - GENERIC_EXPAND_DICT); + if (fsa==null) { + RewriterUtils.error(logger, "Error retrieving FSA dictionary: " + GENERIC_EXPAND_DICT); return false; } // Create Phrase Matcher diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java index e739608387d..b6c886c91d7 100644 --- a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java +++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java @@ -12,6 +12,7 @@ import java.util.Map; * @author Tony Vaagenes */ public class ItemContext { + private class Connectivity { final String id; final double strength; @@ -43,7 +44,8 @@ public class ItemContext { private Item getItem(String id) { Item item = itemById.get(id); if (item == null) - throw new IllegalArgumentException("No item with id '" + id + "'."); + throw new IllegalArgumentException("No item with id '" + id + "'"); return item; } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java index 04c01d7acc1..503987af027 100644 --- a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java +++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java @@ -10,6 +10,7 @@ import com.yahoo.search.query.textserialize.serializer.ItemIdMapper; * @author Tony Vaagenes */ public abstract class TermConverter implements ItemFormConverter { + @Override public Object formToItem(String name, ItemArguments arguments, ItemContext context) { ensureOnlyOneChild(arguments); @@ -22,7 +23,6 @@ public abstract class TermConverter implements ItemFormConverter { abstract TermItem newTermItem(String word); - private void ensureOnlyOneChild(ItemArguments arguments) { if (arguments.children.size() != 1) { throw new IllegalArgumentException("Expected exactly one argument, got '" + @@ -50,4 +50,5 @@ public abstract class TermConverter implements ItemFormConverter { } protected abstract String getValue(TermItem item); + } diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java index b7843a300dc..70eba9d8b50 100644 --- a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java +++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java @@ -7,6 +7,7 @@ import com.yahoo.protect.Validator; * @author Tony Vaagenes */ public class TypeCheck { + public static void ensureInstanceOf(Object object, Class<?> c) { Validator.ensureInstanceOf(expectationString(c.getName(), object.getClass().getSimpleName()), object, c); @@ -24,4 +25,5 @@ public class TypeCheck { private static String expectationString(String expected, String got) { return "Expected " + expected + ", but got " + got; } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java index d4e499bab79..d849e744184 100644 --- a/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java +++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java @@ -14,6 +14,7 @@ import static com.yahoo.search.query.textserialize.item.ListUtil.first; * @author Tony Vaagenes */ class Serializer { + static String serialize(Object child, ItemIdMapper itemIdMapper) { if (child instanceof DispatchForm) { return ((DispatchForm) child).serialize(itemIdMapper); @@ -76,4 +77,5 @@ class Serializer { static String serializeItem(Item item, ItemIdMapper itemIdMapper) { return ItemExecutorRegistry.getByType(item.getItemType()).itemToForm(item, itemIdMapper).serialize(itemIdMapper); } + } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanAttributeParser.java b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanAttributeParser.java index fd76cc4397a..532de1270da 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanAttributeParser.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanAttributeParser.java @@ -14,10 +14,10 @@ import java.math.BigInteger; * a 64-bit hex number <code>0x1234</code> or a list of bits <code>[0, 2, 43, * 22, ...]</code>. * - * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a> - * @since 5.1.15 + * @author Magnar Nedland */ abstract class BooleanAttributeParser extends SimpleMapParser { + private boolean isMap = true; @Override diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java index f77301f587c..de01773c27c 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java @@ -4,6 +4,7 @@ package com.yahoo.search.querytransform; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.prelude.query.PredicateQueryItem; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -26,6 +27,7 @@ import static com.yahoo.yolean.Exceptions.toMessageString; @After({ STEMMING, ACCENT_REMOVAL }) @Provides(BooleanSearcher.PREDICATE) public class BooleanSearcher extends Searcher { + private static final CompoundName FIELD = new CompoundName("boolean.field"); private static final CompoundName ATTRIBUTES = new CompoundName("boolean.attributes"); private static final CompoundName RANGE_ATTRIBUTES = new CompoundName("boolean.rangeAttributes"); @@ -61,7 +63,11 @@ public class BooleanSearcher extends Searcher { } catch (TokenMgrException e) { return new Result(query, ErrorMessage.createInvalidQueryParameter(toMessageString(e))); } - } else { + catch (IllegalArgumentException e) { + throw new IllegalInputException("Failed boolean search on field '" + fieldName + "'", e); + } + } + else { if (query.isTraceable(5)) { query.trace("BooleanSearcher: Nothing added to query", false, 5); } @@ -79,7 +85,9 @@ public class BooleanSearcher extends Searcher { } static public class PredicateValueAttributeParser extends BooleanAttributeParser { - private PredicateQueryItem item; + + private final PredicateQueryItem item; + public PredicateValueAttributeParser(PredicateQueryItem item) { this.item = item; } @@ -93,10 +101,13 @@ public class BooleanSearcher extends Searcher { protected void addAttribute(String attribute, String value, BigInteger subQueryMask) { item.addFeature(attribute, value, subQueryMask.longValue()); } + } static private class PredicateRangeAttributeParser extends BooleanAttributeParser { - private PredicateQueryItem item; + + private final PredicateQueryItem item; + public PredicateRangeAttributeParser(PredicateQueryItem item) { this.item = item; } @@ -110,5 +121,7 @@ public class BooleanSearcher extends Searcher { protected void addAttribute(String attribute, String value, BigInteger subQueryMask) { item.addRangeFeature(attribute, Long.parseLong(value), subQueryMask.longValue()); } + } + } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java index 399ff6194c8..90bcebb53ad 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java @@ -107,7 +107,7 @@ public class NGramSearcher extends Searcher { */ protected Item splitToGrams(Item term, String text, int gramSize, Query query) { String index = ((HasIndexItem)term).getIndexName(); - CompositeItem gramsItem = createGramRoot(query); + CompositeItem gramsItem = createGramRoot((HasIndexItem)term, query); gramsItem.setIndexName(index); Substring origin = ((BlockItem)term).getOrigin(); for (Iterator<GramSplitter.Gram> i = getGramSplitter().split(text,gramSize); i.hasNext(); ) { @@ -130,12 +130,20 @@ public class NGramSearcher extends Searcher { * called by {@link #splitToGrams}. This hook is provided to make it easy to create a subclass which * matches grams using a different composite item, e.g an OrItem. * <p> - * This default implementation return new AndItem(); + * This default implementation returns createGramRoot(query). * + * @param term the term item this gram root is replacing in the query tree, + * typically used to access the index name of the term when that is required by the new gram root + * (such as in PhraseItem) * @param query the input query, to make it possible to return a different composite item type * depending on the query content * @return the composite item to add the gram items to in {@link #splitToGrams} */ + protected CompositeItem createGramRoot(HasIndexItem term, Query query) { + return createGramRoot(query); + } + + /** Creates the root of the query subtree without access to the term being replaced. */ protected CompositeItem createGramRoot(Query query) { return new AndItem(); } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java index 25488aa7bbc..3aa9e59003d 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java @@ -31,8 +31,7 @@ public class VespaLowercasingSearcher extends LowercasingSearcher { public boolean shouldLowercase(WordItem word, IndexFacts.Session indexFacts) { if (word.isLowercased()) return false; - Index index = indexFacts.getIndex(word.getIndexName()); - return index.isLowercase() || index.isAttribute(); + return indexFacts.getIndex(word.getIndexName()).isLowercase(); } @Override @@ -41,8 +40,7 @@ public class VespaLowercasingSearcher extends LowercasingSearcher { StringBuilder sb = new StringBuilder(); sb.append(commonPath).append(".").append(word.getIndexName()); - Index index = indexFacts.getIndex(sb.toString()); - return index.isLowercase() || index.isAttribute(); + return indexFacts.getIndex(sb.toString()).isLowercase(); } } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java index 0b1387a16a2..4745bd23642 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java @@ -3,18 +3,25 @@ package com.yahoo.search.querytransform; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.DotProductItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.WandItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; -import com.yahoo.text.MapParser; import java.util.LinkedHashMap; import java.util.Map; +import com.yahoo.text.SimpleMapParser; import com.yahoo.yolean.Exceptions; /** @@ -64,7 +71,7 @@ public class WandSearcher extends Searcher { private static final CompoundName WAND_THRESHOLD_BOOST_FACTOR = new CompoundName("wand.thresholdBoostFactor"); private final String fieldName; private final WandType wandType; - private final Map<String, Integer> tokens; + private final Map<Object, Integer> tokens; private final int heapSize; private final double scoreThreshold; private final double thresholdBoostFactor; @@ -74,8 +81,14 @@ public class WandSearcher extends Searcher { if (fieldName != null) { String tokens = query.properties().getString(WAND_TOKENS); if (tokens != null) { - wandType = resolveWandType(execution.context().getIndexFacts().newSession(query), query); - this.tokens = new IntegerMapParser().parse(tokens, new LinkedHashMap<>()); + IndexFacts.Session indexFacts = execution.context().getIndexFacts().newSession(query); + Index index = indexFacts.getIndex(fieldName); + wandType = resolveWandType(index, indexFacts, query); + if (index.isNumerical() && (wandType == WandType.DOT_PRODUCT || wandType == WandType.PARALLEL)) { + this.tokens = new LongIntegerMapParser().parse(tokens, new LinkedHashMap<>(200)); + } else { + this.tokens = new MapObjectIntegerParser().parse(tokens, new LinkedHashMap<>(200)); + } heapSize = resolveHeapSize(query); scoreThreshold = resolveScoreThreshold(query); thresholdBoostFactor = resolveThresholdBoostFactor(query); @@ -89,10 +102,9 @@ public class WandSearcher extends Searcher { thresholdBoostFactor = 1; } - private WandType resolveWandType(IndexFacts.Session indexFacts, Query query) { - Index index = indexFacts.getIndex(fieldName); + private WandType resolveWandType(Index index, IndexFacts.Session indexFacts, Query query) { if (index.isNull()) { - throw new IllegalArgumentException("Field '" + fieldName + "' was not found in " + indexFacts); + throw new IllegalInputException("Field '" + fieldName + "' was not found in " + indexFacts); } else { return WandType.create(query.properties().getString(WAND_TYPE, "vespa")); } @@ -100,15 +112,15 @@ public class WandSearcher extends Searcher { private int resolveHeapSize(Query query) { String defaultHeapSize = "100"; - return Integer.valueOf(query.properties().getString(WAND_HEAP_SIZE, defaultHeapSize)); + return Integer.parseInt(query.properties().getString(WAND_HEAP_SIZE, defaultHeapSize)); } private double resolveScoreThreshold(Query query) { - return Double.valueOf(query.properties().getString(WAND_SCORE_THRESHOLD, "0")); + return Double.parseDouble(query.properties().getString(WAND_SCORE_THRESHOLD, "0")); } private double resolveThresholdBoostFactor(Query query) { - return Double.valueOf(query.properties().getString(WAND_THRESHOLD_BOOST_FACTOR, "1")); + return Double.parseDouble(query.properties().getString(WAND_THRESHOLD_BOOST_FACTOR, "1")); } public boolean hasValidData() { @@ -119,7 +131,7 @@ public class WandSearcher extends Searcher { return fieldName; } - public Map<String, Integer> getTokens() { + public Map<Object, Integer> getTokens() { return tokens; } @@ -161,17 +173,17 @@ public class WandSearcher extends Searcher { } else if (inputs.getWandType().equals(WandType.OR)) { return populate(new OrItem(), inputs.getFieldName(), inputs.getTokens()); } else if (inputs.getWandType().equals(WandType.PARALLEL)) { - return populate(new WandItem(inputs.getFieldName(), inputs.getHeapSize()), - inputs.getScoreThreshold(), inputs.getThresholdBoostFactor(), inputs.getTokens()); + return populate(new WandItem(inputs.getFieldName(), inputs.getHeapSize(), inputs.getTokens()), + inputs.getScoreThreshold(), inputs.getThresholdBoostFactor()); } else if (inputs.getWandType().equals(WandType.DOT_PRODUCT)) { - return populate(new DotProductItem(inputs.getFieldName()), inputs.getTokens()); + return new DotProductItem(inputs.getFieldName(), inputs.getTokens()); } - throw new IllegalArgumentException("Unknown type '" + inputs.getWandType() + "'"); + throw new IllegalInputException("Unknown type '" + inputs.getWandType() + "'"); } - private CompositeItem populate(CompositeItem parent, String fieldName, Map<String,Integer> tokens) { - for (Map.Entry<String,Integer> entry : tokens.entrySet()) { - WordItem wordItem = new WordItem(entry.getKey(), fieldName); + private CompositeItem populate(CompositeItem parent, String fieldName, Map<Object,Integer> tokens) { + for (Map.Entry<Object,Integer> entry : tokens.entrySet()) { + WordItem wordItem = new WordItem(entry.getKey().toString(), fieldName); wordItem.setWeight(entry.getValue()); wordItem.setStemmed(true); wordItem.setNormalizable(false); @@ -180,25 +192,34 @@ public class WandSearcher extends Searcher { return parent; } - private WeightedSetItem populate(WeightedSetItem item, Map<String,Integer> tokens) { - for (Map.Entry<String,Integer> entry : tokens.entrySet()) { - item.addToken(entry.getKey(), entry.getValue()); - } - return item; - } - - private WandItem populate(WandItem item, double scoreThreshold, double thresholdBoostFactor, Map<String,Integer> tokens) { - populate(item, tokens); + private WandItem populate(WandItem item, double scoreThreshold, double thresholdBoostFactor) { item.setScoreThreshold(scoreThreshold); item.setThresholdBoostFactor(thresholdBoostFactor); return item; } - private static class IntegerMapParser extends MapParser<Integer> { + private static class MapObjectIntegerParser extends SimpleMapParser { + protected Map<Object, Integer> map; + + public Map<Object,Integer> parse(String string, Map<Object,Integer> map) { + this.map = map; + parse(string); + return this.map; + } + @Override - protected Integer parseValue(String s) { - return Integer.parseInt(s); + protected void handleKeyValue(String key, String value) { + map.put(key, Integer.parseInt(value)); } } + private static class LongIntegerMapParser extends MapObjectIntegerParser { + + @Override + protected void handleKeyValue(String key, String value) { + map.put(Long.parseLong(key), Integer.parseInt(value)); + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java new file mode 100644 index 00000000000..3a392fcbda4 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -0,0 +1,68 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + +import com.yahoo.prelude.query.*; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * Recursively replaces all instances of OrItems with WeakAndItems if the query property weakand.replace is true. + * Otherwise a noop searcher. + * + * @author karowan + */ +public class WeakAndReplacementSearcher extends Searcher { + private static final CompoundName WEAKAND_REPLACE = new CompoundName("weakAnd.replace"); + + @Override public Result search(Query query, Execution execution) { + if (!query.properties().getBoolean(WEAKAND_REPLACE)) { + return execution.search(query); + } + replaceOrItems(query); + return execution.search(query); + } + + /** + * Extracts the queryTree root and the wand.hits property to send to the recursive replacement function + * @param query the search query + */ + private void replaceOrItems(Query query) { + Item root = query.getModel().getQueryTree().getRoot(); + int hits = query.properties().getInteger("wand.hits", WeakAndItem.defaultN); + query.getModel().getQueryTree().setRoot(replaceOrItems(root, hits)); + if (root != query.getModel().getQueryTree().getRoot()) + query.trace("Replaced OR by WeakAnd", true, 2); + } + + + /** + * Recursively iterates over an Item to replace all instances of OrItems with WeakAndItems + * @param item the current item in the replacement iteration + * @param hits the wand.hits property from the request which is assigned to the N value of the new WeakAndItem + * @return the original item or a WeakAndItem replacement of an OrItem + */ + private Item replaceOrItems(Item item, int hits) { + if (!(item instanceof CompositeItem)) { + return item; + } + CompositeItem compositeItem = (CompositeItem) item; + if (compositeItem instanceof OrItem) { + WeakAndItem newItem = new WeakAndItem(hits); + newItem.setWeight(compositeItem.getWeight()); + compositeItem.items().forEach(newItem::addItem); + compositeItem = newItem; + } + for (int i = 0; i < compositeItem.getItemCount(); i++) { + Item subItem = compositeItem.getItem(i); + Item replacedItem = replaceOrItems(subItem, hits); + if (replacedItem != subItem) { + compositeItem.setItem(i, replacedItem); + } + } + return compositeItem; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 31f8194b3b7..ee0e7f4fe0e 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -45,8 +45,7 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.NanNumber; import com.yahoo.tensor.Tensor; -import org.json.JSONArray; -import org.json.JSONObject; +import com.yahoo.tensor.serialization.JsonFormat; import java.io.IOException; import java.io.OutputStream; @@ -60,13 +59,14 @@ import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collections; import java.util.Deque; -import java.util.Iterator; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.Executor; import java.util.function.LongSupplier; +import static com.fasterxml.jackson.databind.SerializationFeature.FLUSH_AFTER_WRITE_VALUE; + /** * JSON renderer for search results. * @@ -78,6 +78,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug"); private static final CompoundName JSON_CALLBACK = new CompoundName("jsoncallback"); + private static final CompoundName TENSOR_FORMAT = new CompoundName("format.tensors"); // if this must be optimized, simply use com.fasterxml.jackson.core.SerializableString private static final String BUCKET_LIMITS = "limits"; @@ -127,6 +128,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private LongSupplier timeSource; private OutputStream stream; + private boolean tensorShortFormRendering = false; + public JsonRenderer() { this(null); } @@ -149,7 +152,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { * @return an object mapper for the internal JsonFactory */ protected static ObjectMapper createJsonCodec() { - return new ObjectMapper(); + return new ObjectMapper().disable(FLUSH_AFTER_WRITE_VALUE); } @Override @@ -166,6 +169,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { public void beginResponse(OutputStream stream) throws IOException { beginJsonCallback(stream); debugRendering = getDebugRendering(getResult().getQuery()); + tensorShortFormRendering = getTensorShortFormRendering(getResult().getQuery()); setGenerator(generatorFactory.createGenerator(stream, JsonEncoding.UTF8), debugRendering); renderedChildren = new ArrayDeque<>(); generator.writeStartObject(); @@ -200,6 +204,12 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return q != null && q.properties().getBoolean(DEBUG_RENDERING_KEY, false); } + private boolean getTensorShortFormRendering(Query q) { + if (q == null || q.properties().get(TENSOR_FORMAT) == null) + return false; + return q.properties().getString(TENSOR_FORMAT).equalsIgnoreCase("short"); + } + protected void renderTrace(Trace trace) throws IOException { if (!trace.traceNode().children().iterator().hasNext()) return; if (getResult().getQuery().getTraceLevel() == 0) return; @@ -285,8 +295,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { generator.writeEndObject(); } generator.writeEndArray(); - - } protected void renderCoverage() throws IOException { @@ -510,7 +518,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } protected FieldConsumer createFieldConsumer(JsonGenerator generator, boolean debugRendering) { - return new FieldConsumer(generator, debugRendering); + return new FieldConsumer(generator, debugRendering, tensorShortFormRendering); } /** @@ -529,12 +537,18 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private final JsonGenerator generator; private final boolean debugRendering; + private final boolean tensorShortForm; private MutableBoolean hasFieldsField; public FieldConsumer(JsonGenerator generator, boolean debugRendering) { + this(generator, debugRendering, false); + } + + public FieldConsumer(JsonGenerator generator, boolean debugRendering, boolean tensorShortForm) { this.generator = generator; this.debugRendering = debugRendering; + this.tensorShortForm = tensorShortForm; } /** @@ -659,7 +673,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } else if (field instanceof Tensor) { renderTensor(Optional.of((Tensor)field)); } else if (field instanceof FeatureData) { - generator.writeRawValue(((FeatureData)field).toJson()); + generator.writeRawValue(((FeatureData)field).toJson(tensorShortForm)); } else if (field instanceof Inspectable) { renderInspectorDirect(((Inspectable)field).inspect()); } else if (field instanceof JsonProducer) { @@ -671,14 +685,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } else if (field instanceof FieldValue) { // the null below is the field which has already been written ((FieldValue) field).serialize(null, new JsonWriter(generator)); - } else if (field instanceof JSONArray || field instanceof JSONObject) { - // org.json returns null if the object would not result in syntactically correct JSON - String s = field.toString(); - if (s == null) { - generator.writeNull(); - } else { - generator.writeRawValue(s); - } } else { generator.writeString(field.toString()); } @@ -705,26 +711,18 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } private void renderTensor(Optional<Tensor> tensor) throws IOException { - generator.writeStartObject(); - generator.writeArrayFieldStart("cells"); - if (tensor.isPresent()) { - for (Iterator<Tensor.Cell> i = tensor.get().cellIterator(); i.hasNext(); ) { - Tensor.Cell cell = i.next(); - - generator.writeStartObject(); - - generator.writeObjectFieldStart("address"); - for (int d = 0; d < cell.getKey().size(); d++) - generator.writeObjectField(tensor.get().type().dimensions().get(d).name(), cell.getKey().label(d)); - generator.writeEndObject(); - - generator.writeObjectField("value", cell.getValue()); - - generator.writeEndObject(); - } + if (tensor.isEmpty()) { + generator.writeStartObject(); + generator.writeArrayFieldStart("cells"); + generator.writeEndArray(); + generator.writeEndObject(); + return; + } + if (tensorShortForm) { + generator.writeRawValue(new String(JsonFormat.encodeShortForm(tensor.get()), StandardCharsets.UTF_8)); + } else { + generator.writeRawValue(new String(JsonFormat.encode(tensor.get()), StandardCharsets.UTF_8)); } - generator.writeEndArray(); - generator.writeEndObject(); } } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java index 783045babf4..9540bc20bc5 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java @@ -4,6 +4,7 @@ package com.yahoo.search.rendering; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Result; import com.yahoo.search.pagetemplates.result.PageTemplatesXmlRenderer; @@ -103,8 +104,8 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi com.yahoo.processing.rendering.Renderer<Result> renderer = getComponent(format); if (renderer == null) - throw new IllegalArgumentException("No renderer with id or alias '" + format + "'. " + - "Available renderers are: [" + rendererNames() + "]."); + throw new IllegalInputException("No renderer with id or alias '" + format + "'. " + + "Available renderers are: [" + rendererNames() + "]."); return renderer; } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java index 9d0e110a6dd..d79122ac9c4 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java @@ -2,7 +2,7 @@ package com.yahoo.search.rendering; import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.URI; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.hitfield.HitField; @@ -395,7 +395,7 @@ public final class SyncDefaultRenderer extends Renderer { if (e instanceof IOException) { throw (IOException) e; } else { - log.log(LogLevel.WARNING, "Exception thrown when rendering the result:", e); + log.log(Level.WARNING, "Exception thrown when rendering the result:", e); } } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java index 62ee16993fd..f3b0cd3a61e 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java @@ -352,8 +352,8 @@ public final class XmlRenderer extends AsynchronousSectionedRenderer<Result> { try { return (Result) getResponse(); } catch (ClassCastException e) { - throw new IllegalArgumentException("XmlRenderer attempted used outside a search context, got a " + - getResponse().getClass().getName()); + throw new IllegalStateException("XmlRenderer attempted used outside a search context, got a " + + getResponse().getClass().getName()); } } diff --git a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java index 4895db04462..7673352576d 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java +++ b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java @@ -14,7 +14,9 @@ import com.yahoo.tensor.serialization.TypedBinaryFormat; import java.nio.charset.StandardCharsets; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -31,6 +33,10 @@ public class FeatureData implements Inspectable, JsonProducer { private Set<String> featureNames = null; + /** Cached decoded values */ + private Map<String, Double> decodedDoubles = null; + private Map<String, Tensor> decodedTensors = null; + private String jsonForm = null; public FeatureData(Inspector value) { @@ -56,9 +62,17 @@ public class FeatureData implements Inspectable, JsonProducer { return jsonForm; } + public String toJson(boolean tensorShortForm) { + if (this == empty) return "{}"; + if (jsonForm != null) return jsonForm; + + jsonForm = JsonRender.render(value, new Encoder(new StringBuilder(), true, tensorShortForm)).toString(); + return jsonForm; + } + @Override public StringBuilder writeJson(StringBuilder target) { - return JsonRender.render(value, new Encoder(target, true)); + return JsonRender.render(value, new Encoder(target, true, false)); } /** @@ -68,6 +82,19 @@ public class FeatureData implements Inspectable, JsonProducer { * (that is, if it is a tensor with nonzero rank) */ public Double getDouble(String featureName) { + if (decodedDoubles == null) + decodedDoubles = new HashMap<>(); + + Double value = decodedDoubles.get(featureName); + if (value != null) return value; + + value = decodeDouble(featureName); + if (value != null) + decodedDoubles.put(featureName, value); + return value; + } + + private Double decodeDouble(String featureName) { Inspector featureValue = getInspector(featureName); if ( ! featureValue.valid()) return null; @@ -83,6 +110,19 @@ public class FeatureData implements Inspectable, JsonProducer { * This will return any feature value: Scalars are returned as a rank 0 tensor. */ public Tensor getTensor(String featureName) { + if (decodedTensors == null) + decodedTensors = new HashMap<>(); + + Tensor value = decodedTensors.get(featureName); + if (value != null) return value; + + value = decodeTensor(featureName); + if (value != null) + decodedTensors.put(featureName, value); + return value; + } + + private Tensor decodeTensor(String featureName) { Inspector featureValue = getInspector(featureName); if ( ! featureValue.valid()) return null; @@ -130,15 +170,19 @@ public class FeatureData implements Inspectable, JsonProducer { /** A JSON encoder which encodes DATA as a tensor */ private static class Encoder extends JsonRender.StringEncoder { - Encoder(StringBuilder out, boolean compact) { + private final boolean tensorShortForm; + + Encoder(StringBuilder out, boolean compact, boolean tensorShortForm) { super(out, compact); + this.tensorShortForm = tensorShortForm; } @Override public void encodeDATA(byte[] value) { // This could be done more efficiently ... - target().append(new String(JsonFormat.encodeWithType(TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(value))), - StandardCharsets.UTF_8)); + Tensor tensor = TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(value)); + byte[] encodedTensor = tensorShortForm ? JsonFormat.encodeShortForm(tensor) : JsonFormat.encodeWithType(tensor); + target().append(new String(encodedTensor, StandardCharsets.UTF_8)); } } diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index fc416c0d930..d224ec2c7ab 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -35,6 +35,8 @@ import java.util.function.BiConsumer; * done of a lightweight version of the hits, which is cheaper if a significant * number of hits are filtered out.</p> * + * <p>Do not cache this as it holds references to objects that should be garbage collected.</p> + * * @author bratseth */ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hit>, Cloneable { @@ -45,8 +47,8 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi private static final String DOCUMENT_ID = "documentid"; /** A collection of string keyed object properties. */ - private Map<String,Object> fields = null; - private Map<String,Object> unmodifiableFieldMap = null; + private Map<String, Object> fields = null; + private Map<String, Object> unmodifiableFieldMap = null; /** Meta data describing how a given searcher should treat this hit. */ // TODO: The case for this is to allow multiple levels of federation searcher routing. @@ -473,7 +475,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi private Map<String, Object> getFieldMap(int minSize) { if (fields == null) { // Compensate for loadfactor and then some, rounded up.... - fields = new LinkedHashMap<>(2*minSize); + fields = new LinkedHashMap<>(2 * minSize); } return fields; } @@ -505,7 +507,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi } /** Returns the types of this as a modifiable set. Modifications to this set are directly reflected in this hit */ - //TODO This shoudld not be exposed as a modifiable set + // TODO: This should not be exposed as a modifiable set public Set<String> types() { if (types == null) types = new ArraySet<>(1); @@ -682,7 +684,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * Called for fields which are available as UTF-8 instead of accept(String, Object). * * @param fieldName the name of the field - * @param utf8Data raw utf-8 data. The reciver <b>must not</b> modify this data + * @param utf8Data raw utf-8 data. The receiver <b>must not</b> modify this data * @param offset the start index in the utf8Data array of the data to accept * @param length the length starting from offset in the utf8Data array of the data to accept */ diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 84fe88d0292..fac0d35d509 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -6,7 +6,7 @@ import com.yahoo.language.Linguistics; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; -import com.yahoo.prelude.query.parser.SpecialTokenRegistry; +import com.yahoo.language.process.SpecialTokenRegistry; import com.yahoo.processing.Processor; import com.yahoo.processing.Request; import com.yahoo.processing.Response; @@ -17,8 +17,6 @@ import com.yahoo.search.cluster.PingableSearcher; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.statistics.TimeTracker; -import java.util.logging.Logger; - /** * <p>An execution of a search chain. This keeps track of the call state for an execution (in the calling thread) * of the searchers of a search chain.</p> @@ -111,7 +109,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { public Context(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts, SpecialTokenRegistry tokenRegistry, RendererRegistry rendererRegistry, Linguistics linguistics) { - owner=null; + owner = null; // The next time something is added here, compose into wrapper objects. Many arguments... // Four methods need to be updated when adding something: @@ -525,7 +523,6 @@ public class Execution extends com.yahoo.processing.execution.Execution { * * @param result the result to fill */ - @SuppressWarnings("deprecation") public void fillAttributes(Result result) { fill(result, ATTRIBUTEPREFETCH); } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index 31b6d06f78e..a813229c984 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -13,7 +13,7 @@ import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; -import com.yahoo.prelude.query.parser.SpecialTokenRegistry; +import com.yahoo.language.process.SpecialTokenRegistry; import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Searcher; import com.yahoo.search.config.IndexInfoConfig; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java b/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java index ec79979387b..d2809e8fe33 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java @@ -10,6 +10,7 @@ package com.yahoo.search.searchchain; * @author Steinar Knutsen */ public final class PhaseNames { + private PhaseNames() { } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java index cf4f5f360ad..2f680a8f3bd 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java @@ -23,6 +23,7 @@ import java.util.Set; */ @SuppressWarnings({"rawtypes", "unchecked"}) public class VespaSearchers { + public static final Collection<ChainedComponentModel> vespaSearcherModels = toSearcherModels( com.yahoo.prelude.querytransform.PhrasingSearcher.class, @@ -33,8 +34,8 @@ public class VespaSearchers { com.yahoo.prelude.searcher.BlendingSearcher.class, com.yahoo.prelude.searcher.PosSearcher.class, com.yahoo.prelude.semantics.SemanticSearcher.class, - com.yahoo.search.grouping.GroupingQueryParser.class); - + com.yahoo.search.grouping.GroupingQueryParser.class, + com.yahoo.search.querytransform.WeakAndReplacementSearcher.class); public static final Collection<ChainedComponentModel> nativeSearcherModels; @@ -59,8 +60,9 @@ public class VespaSearchers { private static FederationSearcherModel federationSearcherModel() { return new FederationSearcherModel(new ComponentSpecification("federation"), - Dependencies.emptyDependencies(), - Collections.emptyList(), true); + Dependencies.emptyDependencies(), + Collections.emptyList(), + true); } private static boolean allAdded(Collection<ChainedComponentModel> searcherModels, Set<ComponentId> componentIds) { @@ -81,4 +83,5 @@ public class VespaSearchers { } return searcherModels; } + } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationSearcherModel.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationSearcherModel.java index e1c95e9478e..01dccee5c7f 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationSearcherModel.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationSearcherModel.java @@ -5,7 +5,6 @@ import java.util.List; import com.google.common.collect.ImmutableList; import com.yahoo.container.bundle.BundleInstantiationSpecification; -import net.jcip.annotations.Immutable; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.chain.dependencies.Dependencies; @@ -17,24 +16,9 @@ import com.yahoo.search.federation.FederationSearcher; * * @author Tony Vaagenes */ -@Immutable public class FederationSearcherModel extends ChainedComponentModel { - /** - * Specifies one or more search chains that can be addressed - * as a single source. - */ - public static class TargetSpec { - public final ComponentSpecification sourceSpec; - public final FederationOptions federationOptions; - - public TargetSpec(ComponentSpecification sourceSpec, FederationOptions federationOptions) { - this.sourceSpec = sourceSpec; - this.federationOptions = federationOptions; - } - } - - private static ComponentSpecification federationSearcherComponentSpecification = + private static final ComponentSpecification federationSearcherComponentSpecification = new ComponentSpecification(FederationSearcher.class.getName()); public final List<TargetSpec> targets; @@ -48,4 +32,16 @@ public class FederationSearcherModel extends ChainedComponentModel { this.targets = ImmutableList.copyOf(targets); } + /** Specifies one or more search chains that can be addressed as a single source. */ + public static class TargetSpec { + + public final ComponentSpecification sourceSpec; + public final FederationOptions federationOptions; + + public TargetSpec(ComponentSpecification sourceSpec, FederationOptions federationOptions) { + this.sourceSpec = sourceSpec; + this.federationOptions = federationOptions; + } + } + } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java index 4c36ca9b4da..160f917f6c6 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java @@ -10,6 +10,7 @@ import com.yahoo.search.Searcher; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Objects; import net.jcip.annotations.Immutable; @@ -33,6 +34,7 @@ public class LocalProviderSpec { com.yahoo.search.querytransform.RangeQueryOptimizer.class, com.yahoo.search.querytransform.SortingDegrader.class, com.yahoo.prelude.searcher.ValidateSortingSearcher.class, + com.yahoo.search.searchers.QueryValidator.class, com.yahoo.prelude.cluster.ClusterSearcher.class, com.yahoo.search.grouping.GroupingValidator.class, com.yahoo.search.grouping.vespa.GroupingExecutor.class, @@ -48,15 +50,8 @@ public class LocalProviderSpec { public final String clusterName; - // TODO: make this final - public Integer cacheSize; - - public LocalProviderSpec(String clusterName, Integer cacheSize) { - this.clusterName = clusterName; - this.cacheSize = cacheSize; - - if (clusterName == null) - throw new IllegalArgumentException("Missing cluster name."); + public LocalProviderSpec(String clusterName) { + this.clusterName = Objects.requireNonNull(clusterName, "Cluster name cannot be null"); } public static boolean includesType(String type) { diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java index d39a488626b..e346a766738 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java @@ -97,11 +97,10 @@ public class DocumentSourceSearcher extends Searcher { public Result search(Query query, Execution execution) { queryCount++; Result r = unFilledResults.get(getQueryKeyClone(query)); - if (r == null) { + if (r == null) r = defaultFilledResult.clone(); - } else { + else r = r.clone(); - } r.setQuery(query); r.hits().trim(query.getOffset(), query.getHits()); @@ -182,11 +181,8 @@ public class DocumentSourceSearcher extends Searcher { * reset. For testing - not reliable if multiple threads makes * queries simultaneously */ - public int getQueryCount() { - return queryCount; - } + public int getQueryCount() { return queryCount; } + + public void resetQueryCount() { queryCount = 0; } - public void resetQueryCount() { - queryCount=0; - } } diff --git a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java index 018f4fc7aa9..d8391fe08f3 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java @@ -11,7 +11,9 @@ import java.util.ListIterator; import java.util.Map; import java.util.logging.Logger; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; + +import com.yahoo.component.chain.dependencies.Before; import com.yahoo.metrics.simple.Counter; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.prelude.query.CompositeItem; @@ -19,18 +21,22 @@ import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.query.WordItem; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.PhaseNames; +import com.yahoo.yolean.Exceptions; /** - * Check whether the query tree seems to be "well formed". In other words, run heurestics against + * Check whether the query tree seems to be "well formed". In other words, run heuristics against * the input data to see whether the query should sent to the search backend. * * @author Steinar Knutsen */ +@Before(PhaseNames.BACKEND) public class InputCheckingSearcher extends Searcher { private final Counter utfRejections; @@ -50,10 +56,8 @@ public class InputCheckingSearcher extends Searcher { public Result search(Query query, Execution execution) { try { checkQuery(query); - } catch (IllegalArgumentException e) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Rejected query \"" + query.toString() + "\" on cause of: " + e.getMessage()); - } + } catch (IllegalInputException e) { + log.log(Level.FINE, () -> "Rejected query '" + query.toString() + "' on cause of: " + Exceptions.toMessageString(e)); return new Result(query, ErrorMessage.createIllegalQuery(e.getMessage())); } return execution.search(query); @@ -92,8 +96,9 @@ public class InputCheckingSearcher extends Searcher { repeatedCount++; if (repeatedCount >= MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE) { repeatedConsecutiveTermsInPhraseRejections.add(); - throw new IllegalArgumentException("More than " + MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE + - " ocurrences of term '" + current + "' in a row detected in phrase : " + phrase.toString()); + throw new IllegalInputException("More than " + MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE + + " occurrences of term '" + current + + "' in a row detected in phrase : " + phrase.toString()); } } else { repeatedCount = 0; @@ -125,8 +130,8 @@ public class InputCheckingSearcher extends Searcher { if (count != null) { if (count.get() >= MAX_REPEATED_TERMS_IN_PHRASE) { repeatedTermsInPhraseRejections.add(); - throw new IllegalArgumentException("Phrase contains more than " + MAX_REPEATED_TERMS_IN_PHRASE + - " occurrences of term '" + current + "' in phrase : " + phrase.toString()); + throw new IllegalInputException("Phrase contains more than " + MAX_REPEATED_TERMS_IN_PHRASE + + " occurrences of term '" + current + "' in phrase : " + phrase.toString()); } count.inc(); } else { @@ -169,8 +174,8 @@ public class InputCheckingSearcher extends Searcher { return; } utfRejections.add(); - throw new IllegalArgumentException("The user input has been determined to be double encoded UTF-8." - + " Please investigate whether this is a false positive."); + throw new IllegalInputException("The user input has been determined to be double encoded UTF-8." + + " Please investigate whether this is a false positive."); } private int countSingleCharacterUserTerms(Item queryItem) { diff --git a/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java b/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java new file mode 100644 index 00000000000..024f231f524 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java @@ -0,0 +1,59 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers; + +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.HasIndexItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.ToolBox; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.PhaseNames; + +import static com.yahoo.search.grouping.GroupingQueryParser.SELECT_PARAMETER_PARSING; + +/** + * Validation of query operators against the schema which is searched + * + * @author bratseth + */ +@After(SELECT_PARAMETER_PARSING) +@Before(PhaseNames.BACKEND) +public class QueryValidator extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + IndexFacts.Session session = execution.context().getIndexFacts().newSession(query); + ToolBox.visit(new ItemValidator(session), query.getModel().getQueryTree().getRoot()); + return execution.search(query); + } + + private static class ItemValidator extends ToolBox.QueryVisitor { + + IndexFacts.Session session; + + public ItemValidator(IndexFacts.Session session) { + this.session = session; + } + + @Override + public boolean visit(Item item) { + if (item instanceof HasIndexItem) { + String indexName = ((HasIndexItem)item).getIndexName(); + if (session.getIndex(indexName).isTensor()) + throw new IllegalArgumentException("Cannot search '" + indexName + "': It is a tensor field"); + } + return true; + } + + @Override + public void onExit() { } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java index 8cae081cada..d22dd2e6af6 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java @@ -4,48 +4,46 @@ package com.yahoo.search.searchers; import com.google.common.annotations.Beta; -import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NearestNeighborItem; -import com.yahoo.prelude.query.QueryCanonicalizer; import com.yahoo.prelude.query.ToolBox; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.vespa.GroupingExecutor; import com.yahoo.search.query.ranking.RankProperties; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.yolean.chain.After; +import com.yahoo.yolean.chain.Before; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -// This depends on tensors in query.getRanking which are moved to rank.properties during query.prepare() -// Query.prepare is done at the same time as canonicalization (by GroupingExecutor), so use that constraint. -@After(QueryCanonicalizer.queryCanonicalization) - /** * Validates any NearestNeighborItem query items. * * @author arnej */ -@Beta +@Before(GroupingExecutor.COMPONENT_NAME) // Must happen before query.prepare() public class ValidateNearestNeighborSearcher extends Searcher { - private Map<String, TensorType> validAttributes = new HashMap<>(); + private final Map<String, List<TensorType>> validAttributes = new HashMap<>(); public ValidateNearestNeighborSearcher(AttributesConfig attributesConfig) { for (AttributesConfig.Attribute a : attributesConfig.attribute()) { - TensorType tt = null; + if (! validAttributes.containsKey(a.name())) { + validAttributes.put(a.name(), new ArrayList<TensorType>()); + } if (a.datatype() == AttributesConfig.Attribute.Datatype.TENSOR) { - tt = TensorType.fromSpec(a.tensortype()); + TensorType tt = TensorType.fromSpec(a.tensortype()); + validAttributes.get(a.name()).add(tt); } - validAttributes.put(a.name(), tt); } } @@ -56,7 +54,7 @@ public class ValidateNearestNeighborSearcher extends Searcher { } private Optional<ErrorMessage> validate(Query query) { - NNVisitor visitor = new NNVisitor(query.getRanking().getProperties(), validAttributes); + NNVisitor visitor = new NNVisitor(query.getRanking().getProperties(), validAttributes, query); ToolBox.visit(visitor, query.getModel().getQueryTree().getRoot()); return visitor.errorMessage; } @@ -65,26 +63,24 @@ public class ValidateNearestNeighborSearcher extends Searcher { public Optional<ErrorMessage> errorMessage = Optional.empty(); - private RankProperties rankProperties; - private Map<String, TensorType> validAttributes; + private final Map<String, List<TensorType>> validAttributes; + private final Query query; - public NNVisitor(RankProperties rankProperties, Map<String, TensorType> validAttributes) { - this.rankProperties = rankProperties; + public NNVisitor(RankProperties rankProperties, Map<String, List<TensorType>> validAttributes, Query query) { this.validAttributes = validAttributes; + this.query = query; } @Override public boolean visit(Item item) { if (item instanceof NearestNeighborItem) { - validate((NearestNeighborItem) item); + String error = validate((NearestNeighborItem)item); + if (error != null) + errorMessage = Optional.of(ErrorMessage.createIllegalQuery(error)); } return true; } - private void setError(String description) { - errorMessage = Optional.of(ErrorMessage.createIllegalQuery(description)); - } - private static boolean isCompatible(TensorType lhs, TensorType rhs) { return lhs.dimensions().equals(rhs.dimensions()); } @@ -98,50 +94,36 @@ public class ValidateNearestNeighborSearcher extends Searcher { return true; } - private void validate(NearestNeighborItem item) { - int target = item.getTargetNumHits(); - if (target < 1) { - setError(item.toString() + " has invalid targetNumHits"); - return; - } - String qprop = item.getQueryTensorName(); - List<Object> rankPropValList = rankProperties.asMap().get(qprop); - if (rankPropValList == null) { - setError(item.toString() + " query tensor not found"); - return; - } - if (rankPropValList.size() != 1) { - setError(item.toString() + " query tensor does not have a single value"); - return; - } - Object rankPropValue = rankPropValList.get(0); - if (! (rankPropValue instanceof Tensor)) { - setError(item.toString() + " query tensor should be a tensor, was: "+ - (rankPropValue == null ? "null" : rankPropValue.getClass().toString())); - return; + /** Returns an error message if this is invalid, or null if it is valid */ + private String validate(NearestNeighborItem item) { + if (item.getTargetNumHits() < 1) + return item + " has invalid targetHits " + item.getTargetNumHits() + ": Must be >= 1"; + + String queryFeatureName = "query(" + item.getQueryTensorName() + ")"; + Optional<Tensor> queryTensor = query.getRanking().getFeatures().getTensor(queryFeatureName); + if (queryTensor.isEmpty()) + return item + " requires a tensor rank feature " + queryFeatureName + " but this is not present"; + + if ( ! validAttributes.containsKey(item.getIndexName())) { + return item + " field is not an attribute"; } - Tensor qTensor = (Tensor)rankPropValue; - TensorType qTensorType = qTensor.type(); - - String field = item.getIndexName(); - if (validAttributes.containsKey(field)) { - TensorType fTensorType = validAttributes.get(field); - if (fTensorType == null) { - setError(item.toString() + " field is not a tensor"); - return; + List<TensorType> allTensorTypes = validAttributes.get(item.getIndexName()); + for (TensorType fieldType : allTensorTypes) { + if (isDenseVector(fieldType) && isCompatible(fieldType, queryTensor.get().type())) { + return null; } - if (! isCompatible(fTensorType, qTensorType)) { - setError(item.toString() + " field type "+fTensorType+" does not match query tensor type "+qTensorType); - return; + } + for (TensorType fieldType : allTensorTypes) { + if (isDenseVector(fieldType) && ! isCompatible(fieldType, queryTensor.get().type())) { + return item + " field type " + fieldType + " does not match query type " + queryTensor.get().type(); } - if (! isDenseVector(fTensorType)) { - setError(item.toString() + " tensor type "+fTensorType+" is not a dense vector"); - return; + } + for (TensorType fieldType : allTensorTypes) { + if (! isDenseVector(fieldType)) { + return item + " tensor type " + fieldType + " is not a dense vector"; } - } else { - setError(item.toString() + " field is not an attribute"); - return; } + return item + " field is not a tensor"; } @Override diff --git a/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java b/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java index 108e46fb68e..6d4fcdaf8fb 100644 --- a/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java @@ -137,8 +137,8 @@ public class PeakQpsSearcher extends Searcher { useMetaHit = false; propertyName = null; } else { - throw new IllegalArgumentException("Config definition out of sync with implementation." + - " No way to create output for method " + method + "."); + throw new IllegalStateException("Config definition out of sync with implementation." + + " No way to create output for method " + method + "."); } } diff --git a/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java b/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java index 0e1acd66600..f344872955a 100644 --- a/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java @@ -14,16 +14,15 @@ import com.yahoo.search.statistics.TimeTracker.Activity; import com.yahoo.statistics.Statistics; import com.yahoo.statistics.Value; - /** * A searcher which is intended to be useful as a general probe for * measuring time consumption a search chain. * - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ @Before("rawQuery") public class TimingSearcher extends PingableSearcher { + private Value measurements; private final boolean measurePing; private final boolean measureSearch; @@ -140,5 +139,4 @@ public class TimingSearcher extends PingableSearcher { super.deconstruct(); } - } diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index df9f2af0cce..b2791875d27 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -128,8 +128,7 @@ public class FieldFiller extends Searcher { Set<String> summaryFields = result.getQuery().getPresentation().getSummaryFields(); - if (summaryFields.isEmpty() || - summaryClass == null || + if (summaryFields.isEmpty() || summaryClass == null || result.getQuery().properties().getBoolean(FIELD_FILLER_DISABLE)) { return; } diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java index 8b2438e3ec2..209340e0a99 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java +++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java @@ -5,6 +5,7 @@ import com.google.common.annotations.Beta; import com.google.inject.Inject; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -28,6 +29,7 @@ import java.util.logging.Logger; * * @author Steinar Knutsen */ +// TODO: The query model should do this @Beta @Provides(MinimalQueryInserter.EXTERNAL_YQL) @Before(PhaseNames.TRANSFORMED_QUERY) @@ -40,19 +42,22 @@ public class MinimalQueryInserter extends Searcher { private static final CompoundName MAX_HITS = new CompoundName("maxHits"); private static final CompoundName MAX_OFFSET = new CompoundName("maxOffset"); - private static Logger log = Logger.getLogger(MinimalQueryInserter.class.getName()); + private static final Logger log = Logger.getLogger(MinimalQueryInserter.class.getName()); @Inject public MinimalQueryInserter(Linguistics linguistics) { // Warmup is needed to avoid a large 400ms init cost during first execution of yql code. warmup(linguistics); } + public MinimalQueryInserter() { this(new SimpleLinguistics()); } + static boolean warmup() { return warmup(new SimpleLinguistics()); } + private static boolean warmup(Linguistics linguistics) { Query query = new Query("search/?yql=select%20*%20from%20sources%20where%20title%20contains%20'xyz';"); Result result = insertQuery(query, new ParserEnvironment().setLinguistics(linguistics)); @@ -67,6 +72,18 @@ public class MinimalQueryInserter extends Searcher { return true; } + @Override + public Result search(Query query, Execution execution) { + try { + if (query.properties().get(YQL) == null) return execution.search(query); + Result result = insertQuery(query, ParserEnvironment.fromExecutionContext(execution.context())); + return (result == null) ? execution.search(query) : result; + } + catch (IllegalArgumentException e) { + throw new IllegalInputException("Illegal YQL query", e); + } + } + private static Result insertQuery(Query query, ParserEnvironment env) { YqlParser parser = (YqlParser) ParserFactory.newInstance(Query.Type.YQL, env); parser.setQueryParser(false); @@ -82,12 +99,16 @@ public class MinimalQueryInserter extends Searcher { int maxHits = query.properties().getInteger(MAX_HITS); int maxOffset = query.properties().getInteger(MAX_OFFSET); if (parser.getOffset() > maxOffset) { - return new Result(query, ErrorMessage.createInvalidQueryParameter("Requested offset " + parser.getOffset() - + ", but the max offset allowed is " + maxOffset + ".")); + return new Result(query, + ErrorMessage.createInvalidQueryParameter("Requested offset " + parser.getOffset() + + ", but the max offset allowed is " + + maxOffset + ".")); } if (parser.getHits() > maxHits) { - return new Result(query, ErrorMessage.createInvalidQueryParameter("Requested " + parser.getHits() - + " hits returned, but max hits allowed is " + maxHits + ".")); + return new Result(query, + ErrorMessage.createInvalidQueryParameter("Requested " + parser.getHits() + + " hits returned, but max hits allowed is " + + maxHits + ".")); } } query.getModel().getQueryTree().setRoot(newTree.getRoot()); @@ -116,12 +137,4 @@ public class MinimalQueryInserter extends Searcher { return null; } - @Override - public Result search(Query query, Execution execution) { - if (query.properties().get(YQL) == null) return execution.search(query); - - Result result = insertQuery(query, ParserEnvironment.fromExecutionContext(execution.context())); - return (result == null) ? execution.search(query) : result; - } - } diff --git a/container-search/src/main/java/com/yahoo/search/yql/NullItemException.java b/container-search/src/main/java/com/yahoo/search/yql/NullItemException.java index b451b9c85c5..8502af76858 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/NullItemException.java +++ b/container-search/src/main/java/com/yahoo/search/yql/NullItemException.java @@ -4,11 +4,12 @@ package com.yahoo.search.yql; /** * Used to communicate a NullItem has been encountered in the query tree. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ -@SuppressWarnings("serial") public class NullItemException extends RuntimeException { + public NullItemException(String message) { super(message); } + } diff --git a/container-search/src/main/java/com/yahoo/search/yql/OperatorNode.java b/container-search/src/main/java/com/yahoo/search/yql/OperatorNode.java index 431f159db01..f529de32fe9 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/OperatorNode.java +++ b/container-search/src/main/java/com/yahoo/search/yql/OperatorNode.java @@ -13,9 +13,7 @@ import java.util.Map; /** * Represents a use of an operator against concrete arguments. The types of arguments depend on the operator. - * <p> * The extension point of this scheme is the Operator rather than new types of Nodes. - * <p> * Operators SHOULD take a fixed number of arguments -- wrap variable argument counts in Lists. */ final class OperatorNode<T extends Operator> { @@ -164,7 +162,7 @@ final class OperatorNode<T extends Operator> { } // we are aware only of types used in our logical operator trees -- OperatorNode, List, and constant values - private static final Function<Object, Object> COPY = new Function<Object, Object>() { + private static final Function<Object, Object> COPY = new Function<>() { @Override public Object apply(Object input) { if (input instanceof List) { diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramCompileException.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramCompileException.java index 46dfb780e2d..32ab9d682e3 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramCompileException.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramCompileException.java @@ -1,7 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.yql; -class ProgramCompileException extends RuntimeException { +import com.yahoo.processing.IllegalInputException; + +class ProgramCompileException extends IllegalInputException { private Location sourceLocation; @@ -9,27 +11,6 @@ class ProgramCompileException extends RuntimeException { super(message); } - public ProgramCompileException(String message, Object... args) { - super(formatMessage(message, args)); - } - - private static String formatMessage(String message, Object... args) { - return args == null ? message : String.format(message, args); - } - - public ProgramCompileException(String message, Throwable cause) { - super(message, cause); - } - - public ProgramCompileException(Throwable cause) { - super(cause); - } - - public ProgramCompileException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } - - public ProgramCompileException(Location sourceLocation, String message, Object... args) { super(String.format("%s %s", sourceLocation != null ? sourceLocation : "", args == null ? message : String.format(message, args))); this.sourceLocation = sourceLocation; diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index a270d935a97..d3f07bae428 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -14,76 +14,46 @@ import com.yahoo.search.yql.yqlplusParser.AnnotateExpressionContext; import com.yahoo.search.yql.yqlplusParser.ArgumentContext; import com.yahoo.search.yql.yqlplusParser.ArgumentsContext; import com.yahoo.search.yql.yqlplusParser.ArrayLiteralContext; -import com.yahoo.search.yql.yqlplusParser.ArrayTypeContext; import com.yahoo.search.yql.yqlplusParser.Call_sourceContext; import com.yahoo.search.yql.yqlplusParser.ConstantArrayContext; import com.yahoo.search.yql.yqlplusParser.ConstantExpressionContext; import com.yahoo.search.yql.yqlplusParser.ConstantMapExpressionContext; import com.yahoo.search.yql.yqlplusParser.ConstantPropertyNameAndValueContext; -import com.yahoo.search.yql.yqlplusParser.Delete_statementContext; import com.yahoo.search.yql.yqlplusParser.DereferencedExpressionContext; import com.yahoo.search.yql.yqlplusParser.EqualityExpressionContext; import com.yahoo.search.yql.yqlplusParser.ExpressionContext; -import com.yahoo.search.yql.yqlplusParser.FallbackContext; import com.yahoo.search.yql.yqlplusParser.Field_defContext; -import com.yahoo.search.yql.yqlplusParser.Field_names_specContext; -import com.yahoo.search.yql.yqlplusParser.Field_values_group_specContext; -import com.yahoo.search.yql.yqlplusParser.Field_values_specContext; import com.yahoo.search.yql.yqlplusParser.IdentContext; -import com.yahoo.search.yql.yqlplusParser.Import_listContext; -import com.yahoo.search.yql.yqlplusParser.Import_statementContext; import com.yahoo.search.yql.yqlplusParser.InNotInTargetContext; -import com.yahoo.search.yql.yqlplusParser.Insert_sourceContext; -import com.yahoo.search.yql.yqlplusParser.Insert_statementContext; -import com.yahoo.search.yql.yqlplusParser.Insert_valuesContext; -import com.yahoo.search.yql.yqlplusParser.JoinExpressionContext; -import com.yahoo.search.yql.yqlplusParser.Join_exprContext; import com.yahoo.search.yql.yqlplusParser.LimitContext; import com.yahoo.search.yql.yqlplusParser.Literal_elementContext; import com.yahoo.search.yql.yqlplusParser.Literal_listContext; import com.yahoo.search.yql.yqlplusParser.LogicalANDExpressionContext; import com.yahoo.search.yql.yqlplusParser.LogicalORExpressionContext; import com.yahoo.search.yql.yqlplusParser.MapExpressionContext; -import com.yahoo.search.yql.yqlplusParser.MapTypeContext; -import com.yahoo.search.yql.yqlplusParser.Merge_componentContext; -import com.yahoo.search.yql.yqlplusParser.Merge_statementContext; -import com.yahoo.search.yql.yqlplusParser.ModuleIdContext; -import com.yahoo.search.yql.yqlplusParser.ModuleNameContext; import com.yahoo.search.yql.yqlplusParser.MultiplicativeExpressionContext; import com.yahoo.search.yql.yqlplusParser.Namespaced_nameContext; -import com.yahoo.search.yql.yqlplusParser.Next_statementContext; import com.yahoo.search.yql.yqlplusParser.OffsetContext; import com.yahoo.search.yql.yqlplusParser.OrderbyContext; import com.yahoo.search.yql.yqlplusParser.Orderby_fieldContext; import com.yahoo.search.yql.yqlplusParser.Output_specContext; -import com.yahoo.search.yql.yqlplusParser.Paged_clauseContext; -import com.yahoo.search.yql.yqlplusParser.ParamsContext; import com.yahoo.search.yql.yqlplusParser.Pipeline_stepContext; -import com.yahoo.search.yql.yqlplusParser.Procedure_argumentContext; -import com.yahoo.search.yql.yqlplusParser.Program_arglistContext; import com.yahoo.search.yql.yqlplusParser.Project_specContext; import com.yahoo.search.yql.yqlplusParser.ProgramContext; import com.yahoo.search.yql.yqlplusParser.PropertyNameAndValueContext; import com.yahoo.search.yql.yqlplusParser.Query_statementContext; import com.yahoo.search.yql.yqlplusParser.RelationalExpressionContext; import com.yahoo.search.yql.yqlplusParser.RelationalOpContext; -import com.yahoo.search.yql.yqlplusParser.Returning_specContext; import com.yahoo.search.yql.yqlplusParser.Scalar_literalContext; -import com.yahoo.search.yql.yqlplusParser.Select_source_joinContext; import com.yahoo.search.yql.yqlplusParser.Select_source_multiContext; import com.yahoo.search.yql.yqlplusParser.Select_statementContext; -import com.yahoo.search.yql.yqlplusParser.Selectvar_statementContext; import com.yahoo.search.yql.yqlplusParser.Sequence_sourceContext; import com.yahoo.search.yql.yqlplusParser.Source_listContext; import com.yahoo.search.yql.yqlplusParser.Source_specContext; import com.yahoo.search.yql.yqlplusParser.Source_statementContext; import com.yahoo.search.yql.yqlplusParser.StatementContext; import com.yahoo.search.yql.yqlplusParser.TimeoutContext; -import com.yahoo.search.yql.yqlplusParser.TypenameContext; import com.yahoo.search.yql.yqlplusParser.UnaryExpressionContext; -import com.yahoo.search.yql.yqlplusParser.Update_statementContext; -import com.yahoo.search.yql.yqlplusParser.Update_valuesContext; -import com.yahoo.search.yql.yqlplusParser.ViewContext; import com.yahoo.search.yql.yqlplusParser.WhereContext; import org.antlr.v4.runtime.BaseErrorListener; @@ -126,7 +96,6 @@ final class ProgramParser { return prepareParser(file.getAbsoluteFile().toString(), new CaseInsensitiveFileStream(file.getAbsolutePath())); } - private yqlplusParser prepareParser(String programName, CharStream input) { yqlplusLexer lexer = new yqlplusLexer(input); lexer.removeErrorListeners(); @@ -139,7 +108,7 @@ final class ProgramParser { int charPositionInLine, @NotNull String msg, @Nullable RecognitionException e) { - throw new ProgramCompileException(new Location(programName, line, charPositionInLine), msg); + throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg); } }); @@ -156,7 +125,7 @@ final class ProgramParser { int charPositionInLine, @NotNull String msg, @Nullable RecognitionException e) { - throw new ProgramCompileException(new Location(programName, line, charPositionInLine), msg); + throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg); } }); @@ -168,41 +137,18 @@ final class ProgramParser { try { return parser.program(); } catch (RecognitionException e) { - //Retry parsing using full LL mode + // Retry parsing using full LL mode parser.reset(); parser.getInterpreter().setPredictionMode(PredictionMode.LL); return parser.program(); } } - public OperatorNode<StatementOperator> parse(String programName, InputStream program) throws IOException, RecognitionException { - yqlplusParser parser = prepareParser(programName, program); - return convertProgram(parseProgram(parser), parser, programName); - } - public OperatorNode<StatementOperator> parse(String programName, String program) throws IOException, RecognitionException { yqlplusParser parser = prepareParser(programName, program); return convertProgram(parseProgram(parser), parser, programName); } - public OperatorNode<StatementOperator> parse(File input) throws IOException, RecognitionException { - yqlplusParser parser = prepareParser(input); - return convertProgram(parseProgram(parser), parser, input.getAbsoluteFile().toString()); - } - - public OperatorNode<ExpressionOperator> parseExpression(String input) throws IOException, RecognitionException { - return convertExpr(prepareParser("<expression>", input).expression(false).getRuleContext(), new Scope()); - } - - public OperatorNode<ExpressionOperator> parseExpression(String input, Set<String> visibleAliases) throws IOException, RecognitionException { - Scope scope = new Scope(); - final Location loc = new Location("<expression>", -1, -1); - for (String alias : visibleAliases) { - scope.defineDataSource(loc, alias); - } - return convertExpr(prepareParser("<expression>", input).expression(false).getRuleContext(), scope); - } - private Location toLocation(Scope scope, ParseTree node) { Token start; if (node instanceof ParserRuleContext) { @@ -212,8 +158,7 @@ final class ProgramParser { } else { throw new ProgramCompileException("Location is not available for type " + node.getClass()); } - Location location = new Location(scope != null? scope.programName: "<string>", start.getLine(), start.getCharPositionInLine()); - return location; + return new Location(scope != null? scope.programName: "<string>", start.getLine(), start.getCharPositionInLine()); } private List<String> readName(Namespaced_nameContext node) { @@ -230,14 +175,6 @@ final class ProgramParser { private final List<String> binding; - Binding(String moduleName, String exportName) { - this.binding = ImmutableList.of(moduleName, exportName); - } - - Binding(String moduleName) { - this.binding = ImmutableList.of(moduleName); - } - Binding(List<String> binding) { this.binding = binding; } @@ -263,13 +200,6 @@ final class ProgramParser { final yqlplusParser parser; final String programName; - Scope() { - this.parser = null; - this.programName = null; - this.root = this; - this.parent = null; - } - Scope(yqlplusParser parser, String programName) { this.parser = parser; this.programName = programName; @@ -288,15 +218,10 @@ final class ProgramParser { return parser; } - public String getProgramName() { - return programName; - } - public Set<String> getCursors() { return cursors; } - boolean isBound(String name) { // bindings live only in the 'root' node return root.bindings.containsKey(name); @@ -329,13 +254,6 @@ final class ProgramParser { root.bindings.put(symbolName, new Binding(binding)); } - public void bindModuleSymbol(Location loc, List<String> moduleName, String exportName, String symbolName) { - ImmutableList.Builder<String> builder = ImmutableList.builder(); - builder.addAll(moduleName); - builder.add(exportName); - bindModule(loc, builder.build(), symbolName); - } - public void defineDataSource(Location loc, String name) { if (isCursor(name)) { throw new ProgramCompileException(loc, "Alias '%s' is already used.", name); @@ -376,13 +294,12 @@ final class ProgramParser { } } - private OperatorNode<SequenceOperator> convertSelectOrInsertOrUpdateOrDelete(ParseTree node, Scope scopeParent) { + private OperatorNode<SequenceOperator> convertSelect(ParseTree node, Scope scopeParent) { - Preconditions.checkArgument(node instanceof Select_statementContext || node instanceof Insert_statementContext || - node instanceof Update_statementContext || node instanceof Delete_statementContext); + Preconditions.checkArgument(node instanceof Select_statementContext); - // SELECT^ select_field_spec select_source where? orderby? limit? offset? timeout? fallback? - // select is the only place to define where/orderby/limit/offset and joins + // SELECT^ select_field_spec select_source where? orderby? limit? offset? timeout? + // select is the only place to define where/orderby/limit/offset Scope scope = scopeParent.child(); ProjectionBuilder proj = null; OperatorNode<SequenceOperator> source = null; @@ -391,29 +308,18 @@ final class ProgramParser { OperatorNode<ExpressionOperator> offset = null; OperatorNode<ExpressionOperator> limit = null; OperatorNode<ExpressionOperator> timeout = null; - OperatorNode<SequenceOperator> fallback = null; - OperatorNode<SequenceOperator> insertValues = null; - OperatorNode<ExpressionOperator> updateValues = null; - - ParseTree sourceNode; - if (node instanceof Select_statementContext ) { - sourceNode = node.getChild(2) != null ? node.getChild(2).getChild(0):null; - } else { - sourceNode = node.getChild(1); - } + ParseTree sourceNode = node.getChild(2) != null ? node.getChild(2).getChild(0):null; if (sourceNode != null) { switch (getParseTreeIndex(sourceNode)) { // ALL_SOURCE and MULTI_SOURCE are how FROM SOURCES // *|source_name,... are parsed - // They can't be used directly with the JOIN syntax at this time - case yqlplusParser.RULE_select_source_all: { + case yqlplusParser.RULE_select_source_all: Location location = toLocation(scope, sourceNode.getChild(2)); source = OperatorNode.create(location, SequenceOperator.ALL); source.putAnnotation("alias", "row"); scope.defineDataSource(location, "row"); - } break; case yqlplusParser.RULE_select_source_multi: Source_listContext multiSourceContext = ((Select_source_multiContext) sourceNode).source_list(); @@ -421,22 +327,8 @@ final class ProgramParser { source.putAnnotation("alias", "row"); scope.defineDataSource(toLocation(scope, multiSourceContext), "row"); break; - case yqlplusParser.RULE_select_source_join: + case yqlplusParser.RULE_select_source_from: source = convertSource((ParserRuleContext) sourceNode.getChild(1), scope); - List<Join_exprContext> joinContexts = ((Select_source_joinContext)sourceNode).join_expr(); - for (Join_exprContext joinContext:joinContexts) { - source = convertJoin(joinContext, source, scope); - } - break; - case yqlplusParser.RULE_insert_source: - Insert_sourceContext insertSourceContext = (Insert_sourceContext) sourceNode; - source = convertSource((ParserRuleContext)insertSourceContext.getChild(1), scope); - break; - case yqlplusParser.RULE_delete_source: - source = convertSource((ParserRuleContext)sourceNode.getChild(1), scope); - break; - case yqlplusParser.RULE_update_source: - source = convertSource((ParserRuleContext)sourceNode.getChild(0), scope); break; } } else { @@ -451,9 +343,6 @@ final class ProgramParser { proj = readProjection(((Project_specContext) child.getChild(0)).field_def(), scope); } break; - case yqlplusParser.RULE_returning_spec: - proj = readProjection(((Returning_specContext) child).select_field_spec().project_spec().field_def(), scope); - break; case yqlplusParser.RULE_where: filter = convertExpr(((WhereContext) child).expression(), scope); break; @@ -475,23 +364,6 @@ final class ProgramParser { case yqlplusParser.RULE_timeout: timeout = convertExpr(((TimeoutContext) child).fixed_or_parameter(), scope); break; - case yqlplusParser.RULE_fallback: - fallback = convertQuery(((FallbackContext) child).select_statement(), scope); - break; - case yqlplusParser.RULE_insert_values: - if (child.getChild(0) instanceof yqlplusParser.Query_statementContext) { - insertValues = convertQuery(child.getChild(0).getChild(0), scope); - } else { - insertValues = readBatchValues(((Insert_valuesContext) child).field_names_spec(), ((Insert_valuesContext)child).field_values_group_spec(), scope); - } - break; - case yqlplusParser.RULE_update_values: - if (getParseTreeIndex(child.getChild(0)) == yqlplusParser.RULE_field_def) { - updateValues = readValues(((Update_valuesContext)child).field_def(), scope); - } else { - updateValues = readValues((Field_names_specContext)child.getChild(0), (Field_values_specContext)child.getChild(2), scope); - } - break; } } // now assemble the logical plan @@ -500,26 +372,6 @@ final class ProgramParser { if (filter != null) { result = OperatorNode.create(SequenceOperator.FILTER, result, filter); } - // insert values - if (insertValues != null) { - result = OperatorNode.create(SequenceOperator.INSERT, result, insertValues); - } - // update - if (updateValues != null) { - if (filter != null) { - result = OperatorNode.create(SequenceOperator.UPDATE, source, updateValues, filter); - } else { - result = OperatorNode.create(SequenceOperator.UPDATE_ALL, source, updateValues); - } - } - // delete - if (getParseTreeIndex(node) == yqlplusParser.RULE_delete_statement) { - if (filter != null) { - result = OperatorNode.create(SequenceOperator.DELETE, source, filter); - } else { - result = OperatorNode.create(SequenceOperator.DELETE_ALL, source); - } - } // then sort (or project and sort) boolean projectBeforeSort = false; if (orderby != null) { @@ -558,30 +410,9 @@ final class ProgramParser { if (timeout != null) { result = OperatorNode.create(SequenceOperator.TIMEOUT, result, timeout); } - // if there's a fallback, emit a fallback node - if (fallback != null) { - result = OperatorNode.create(SequenceOperator.FALLBACK, result, fallback); - } return result; } - private OperatorNode<ExpressionOperator> readValues(List<Field_defContext> fieldDefs, Scope scope) { - List<String> fieldNames; - List<OperatorNode<ExpressionOperator>> fieldValues; - int numPairs = fieldDefs.size(); - fieldNames = Lists.newArrayListWithExpectedSize(numPairs); - fieldValues = Lists.newArrayListWithExpectedSize(numPairs); - for (int j = 0; j < numPairs; j++) { - ParseTree startNode = fieldDefs.get(j); - while(startNode.getChildCount() < 3) { - startNode = startNode.getChild(0); - } - fieldNames.add((String) convertExpr(startNode.getChild(0), scope).getArgument(1)); - fieldValues.add(convertExpr(startNode.getChild(2), scope)); - } - return OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues); - } - private OperatorNode<SequenceOperator> readMultiSource(Scope scope, Source_listContext multiSource) { List<List<String>> sourceNameList = Lists.newArrayList(); List<Namespaced_nameContext> nameSpaces = multiSource.namespaced_name(); @@ -591,9 +422,7 @@ final class ProgramParser { } return OperatorNode.create(toLocation(scope, multiSource), SequenceOperator.MULTISOURCE, sourceNameList); } -// pipeline_step -// : namespaced_name arguments[false]? -// ; + private OperatorNode<SequenceOperator> convertPipe(Query_statementContext queryStatementContext, List<Pipeline_stepContext> nodes, Scope scope) { OperatorNode<SequenceOperator> result = convertQuery(queryStatementContext.getChild(0), scope.getRoot()); for (Pipeline_stepContext step:nodes) { @@ -603,7 +432,7 @@ final class ProgramParser { } else { List<String> name = readName(step.namespaced_name()); List<OperatorNode<ExpressionOperator>> args = ImmutableList.of(); - //LPAREN (argument[$in_select] (COMMA argument[$in_select])*) RPAREN + // LPAREN (argument[$in_select] (COMMA argument[$in_select])*) RPAREN if (step.getChildCount() > 1) { ArgumentsContext arguments = step.arguments(); if (arguments.getChildCount() > 2) { @@ -621,56 +450,25 @@ final class ProgramParser { return result; } - private OperatorNode<SequenceOperator> convertMerge(List<Merge_componentContext> mergeComponentList, Scope scope) { - Preconditions.checkArgument(mergeComponentList != null); - List<OperatorNode<SequenceOperator>> sources = Lists.newArrayListWithExpectedSize(mergeComponentList.size()); - for (Merge_componentContext mergeComponent:mergeComponentList) { - Select_statementContext selectContext = mergeComponent.select_statement(); - Source_statementContext sourceContext = mergeComponent.source_statement(); - if (selectContext != null) { - sources.add(convertQuery(selectContext, scope.getRoot())); - } else { - sources.add(convertQuery(sourceContext, scope.getRoot())); - } - } - return OperatorNode.create(SequenceOperator.MERGE, sources); - } - private OperatorNode<SequenceOperator> convertQuery(ParseTree node, Scope scope) { - if (node instanceof Select_statementContext - || node instanceof Insert_statementContext - || node instanceof Update_statementContext - || node instanceof Delete_statementContext) { - return convertSelectOrInsertOrUpdateOrDelete(node, scope.getRoot()); - } else if (node instanceof Source_statementContext) { //for pipe + if (node instanceof Select_statementContext) { + return convertSelect(node, scope.getRoot()); + } else if (node instanceof Source_statementContext) { // for pipe Source_statementContext sourceStatementContext = (Source_statementContext)node; return convertPipe(sourceStatementContext.query_statement(), sourceStatementContext.pipeline_step(), scope); - } else if (node instanceof Merge_statementContext) { - return convertMerge(((Merge_statementContext)node).merge_component(), scope); } else { throw new IllegalArgumentException("Unexpected argument type to convertQueryStatement: " + node.toStringTree()); } } - private OperatorNode<SequenceOperator> convertJoin(Join_exprContext node, OperatorNode<SequenceOperator> left, Scope scope) { - Source_specContext sourceSpec = node.source_spec(); - OperatorNode<SequenceOperator> right = convertSource(sourceSpec, scope); - JoinExpressionContext joinContext = node.joinExpression(); - OperatorNode<ExpressionOperator> joinExpression = readBinOp(ExpressionOperator.valueOf("EQ"), joinContext.getChild(0), joinContext.getChild(2), scope); - if (joinExpression.getOperator() != ExpressionOperator.EQ) { - throw new ProgramCompileException(joinExpression.getLocation(), "Unexpected join expression type: %s (expected EQ)", joinExpression.getOperator()); - } - return OperatorNode.create(toLocation(scope, sourceSpec), node.join_spec().LEFT() != null ? SequenceOperator.LEFT_JOIN : SequenceOperator.JOIN, left, right, joinExpression); - } - private String assignAlias(String alias, ParserRuleContext node, Scope scope) { if (alias == null) { alias = "source"; } - if (node != null && node instanceof yqlplusParser.Alias_defContext) { - //alias_def : (AS? ID); + if (node instanceof yqlplusParser.Alias_defContext) { + // alias_def : (AS? ID); ParseTree idChild = node; if (node.getChildCount() > 1) { idChild = node.getChild(1); @@ -690,20 +488,14 @@ final class ProgramParser { scope.defineDataSource(null, candidate); return alias; } - } - - private OperatorNode<SequenceOperator> convertSource(ParserRuleContext sourceSpecNode, Scope scope) { + } + private OperatorNode<SequenceOperator> convertSource(ParserRuleContext sourceSpecNode, Scope scope) { // DataSources String alias; OperatorNode<SequenceOperator> result; ParserRuleContext dataSourceNode = sourceSpecNode; ParserRuleContext aliasContext = null; - //data_source - //: call_source - //| LPAREN source_statement RPAREN - //| sequence_source - //; if (sourceSpecNode instanceof Source_specContext) { dataSourceNode = (ParserRuleContext)sourceSpecNode.getChild(0); if (sourceSpecNode.getChildCount() == 2) { @@ -717,7 +509,6 @@ final class ProgramParser { } } switch (getParseTreeIndex(dataSourceNode)) { - case yqlplusParser.RULE_write_data_source: case yqlplusParser.RULE_call_source: { List<String> names = readName(dataSourceNode.getChild(Namespaced_nameContext.class, 0)); alias = assignAlias(names.get(names.size() - 1), aliasContext, scope); @@ -763,60 +554,9 @@ final class ProgramParser { return result; } - private OperatorNode<TypeOperator> decodeType(Scope scope, TypenameContext type) { - - TypeOperator op; - ParseTree typeNode = type.getChild(0); - switch (getParseTreeIndex(typeNode)) { - case yqlplusParser.TYPE_BOOLEAN: - op = TypeOperator.BOOLEAN; - break; - case yqlplusParser.TYPE_BYTE: - op = TypeOperator.BYTE; - break; - case yqlplusParser.TYPE_DOUBLE: - op = TypeOperator.DOUBLE; - break; - case yqlplusParser.TYPE_INT16: - op = TypeOperator.INT16; - break; - case yqlplusParser.TYPE_INT32: - op = TypeOperator.INT32; - break; - case yqlplusParser.TYPE_INT64: - op = TypeOperator.INT64; - break; - case yqlplusParser.TYPE_STRING: - op = TypeOperator.STRING; - break; - case yqlplusParser.TYPE_TIMESTAMP: - op = TypeOperator.TIMESTAMP; - break; - case yqlplusParser.RULE_arrayType: - return OperatorNode.create(toLocation(scope, typeNode), TypeOperator.ARRAY, decodeType(scope, ((ArrayTypeContext)typeNode).getChild(TypenameContext.class, 0))); - case yqlplusParser.RULE_mapType: - return OperatorNode.create(toLocation(scope, typeNode), TypeOperator.MAP, decodeType(scope, ((MapTypeContext)typeNode).getChild(TypenameContext.class, 0))); - default: - throw new ProgramCompileException("Unknown type " + typeNode.getText()); - } - return OperatorNode.create(toLocation(scope, typeNode), op); - } - - private List<String> createBindingName(ParseTree node) { - if (node instanceof ModuleNameContext) { - if (((ModuleNameContext)node).namespaced_name() != null) { - return readName(((ModuleNameContext)node).namespaced_name()); - } else if (((ModuleNameContext)node).literalString() != null) { - return ImmutableList.of(((ModuleNameContext)node).literalString().STRING().getText()); - } - } else if (node instanceof ModuleIdContext) { - return ImmutableList.of(node.getText()); - } - throw new ProgramCompileException("Wrong context"); - } - - private OperatorNode<StatementOperator> convertProgram( - ParserRuleContext program, yqlplusParser parser, String programName) { + private OperatorNode<StatementOperator> convertProgram(ParserRuleContext program, + yqlplusParser parser, + String programName) { Scope scope = new Scope(parser, programName); List<OperatorNode<StatementOperator>> stmts = Lists.newArrayList(); int output = 0; @@ -825,148 +565,37 @@ final class ProgramParser { continue; } ParserRuleContext ruleContext = (ParserRuleContext) node; - switch (ruleContext.getRuleIndex()) { - case yqlplusParser.RULE_params: { - // ^(ARGUMENT ident typeref expression?) - ParamsContext paramsContext = (ParamsContext) ruleContext; - Program_arglistContext program_arglistContext = paramsContext.program_arglist(); - if (program_arglistContext != null) { - List<Procedure_argumentContext> argList = program_arglistContext.procedure_argument(); - for (Procedure_argumentContext procedureArgumentContext : argList) { - String name = procedureArgumentContext.ident().getText(); - OperatorNode<TypeOperator> type = decodeType(scope, procedureArgumentContext.getChild(TypenameContext.class, 0)); - OperatorNode<ExpressionOperator> defaultValue = OperatorNode.create(ExpressionOperator.NULL); - if (procedureArgumentContext.expression() != null) { - defaultValue = convertExpr(procedureArgumentContext.expression(), scope); - } - scope.defineVariable(toLocation(scope, procedureArgumentContext), name); - stmts.add(OperatorNode.create(StatementOperator.ARGUMENT, name, type, defaultValue)); - } - } - break; - } - case yqlplusParser.RULE_import_statement: { - Import_statementContext importContext = (Import_statementContext) ruleContext; - if (null == importContext.import_list()) { - List<String> name = createBindingName(node.getChild(1)); - String target; - Location location = toLocation(scope, node.getChild(1)); - if (node.getChildCount() == 2) { - target = name.get(0); - } else if (node.getChildCount() == 4) { - target = node.getChild(3).getText(); - } else { - throw new ProgramCompileException("Unknown node count for IMPORT: " + node.toStringTree()); - } - scope.bindModule(location, name, target); - } else { - // | FROM moduleName IMPORT import_list -> ^(IMPORT_FROM - // moduleName import_list+) - Import_listContext importListContext = importContext.import_list(); - List<String> name = createBindingName(importContext.moduleName()); - Location location = toLocation(scope, importContext.moduleName()); - List<ModuleIdContext> moduleIds = importListContext.moduleId(); - List<String> symbols = Lists.newArrayListWithExpectedSize(moduleIds.size()); - for (ModuleIdContext cnode : moduleIds) { - symbols.add(cnode.ID().getText()); - } - for (String sym : symbols) { - scope.bindModuleSymbol(location, name, sym, sym); - } - } - break; - } + if (ruleContext.getRuleIndex() != yqlplusParser.RULE_statement) + throw new ProgramCompileException("Unknown program element: " + node.getText()); - // DDL - case yqlplusParser.RULE_ddl: - ruleContext = (ParserRuleContext)ruleContext.getChild(0); - break; - case yqlplusParser.RULE_view: { - // view and projection expansion now has to be done by the - // execution engine - // since views/projections, in order to be useful, have to - // support being used from outside the same program - ViewContext viewContext = (ViewContext) ruleContext; - Location loc = toLocation(scope, viewContext); - scope.getRoot().defineView(loc, viewContext.ID().getText()); - stmts.add(OperatorNode.create(loc, StatementOperator.DEFINE_VIEW, viewContext.ID().getText(), convertQuery(viewContext.source_statement(), scope.getRoot()))); - break; + // ^(STATEMENT_QUERY source_statement paged_clause? output_spec?) + StatementContext statementContext = (StatementContext) ruleContext; + Source_statementContext source_statement = statementContext.output_statement().source_statement(); + OperatorNode<SequenceOperator> query; + if (source_statement.getChildCount() == 1) { + query = convertQuery( source_statement.query_statement().getChild(0), scope); + } else { + query = convertQuery(source_statement, scope); } - case yqlplusParser.RULE_statement: { - // ^(STATEMENT_QUERY source_statement paged_clause? - // output_spec?) - StatementContext statementContext = (StatementContext) ruleContext; - switch (getParseTreeIndex(ruleContext.getChild(0))) { - case yqlplusParser.RULE_selectvar_statement: { - // ^(STATEMENT_SELECTVAR ident source_statement) - Selectvar_statementContext selectVarContext = (Selectvar_statementContext) ruleContext.getChild(0); - String variable = selectVarContext.ident().getText(); - OperatorNode<SequenceOperator> query = convertQuery(selectVarContext.source_statement(), scope); - Location location = toLocation(scope, selectVarContext.ident()); - scope.defineVariable(location, variable); - stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable)); - break; + String variable = "result" + (++output); + boolean isCountVariable = false; + ParseTree outputStatement = node.getChild(0); + Location location = toLocation(scope, outputStatement); + for (int i = 1; i < outputStatement.getChildCount(); ++i) { + ParseTree child = outputStatement.getChild(i); + if ( getParseTreeIndex(child) != yqlplusParser.RULE_output_spec) + throw new ProgramCompileException( "Unknown statement attribute: " + child.toStringTree()); + + Output_specContext outputSpecContext = (Output_specContext) child; + variable = outputSpecContext.ident().getText(); + if (outputSpecContext.COUNT() != null) { + isCountVariable = true; } - case yqlplusParser.RULE_next_statement: { - // NEXT^ literalString OUTPUT! AS! ident - Next_statementContext nextStateContext = (Next_statementContext) ruleContext.getChild(0); - String continuationValue = StringUnescaper.unquote(nextStateContext.literalString().getText()); - String variable = nextStateContext.ident().getText(); - Location location = toLocation(scope, node); - OperatorNode<SequenceOperator> next = OperatorNode.create(location, SequenceOperator.NEXT, continuationValue); - stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, next, variable)); - stmts.add(OperatorNode.create(location, StatementOperator.OUTPUT, variable)); - scope.defineVariable(location, variable); - break; - } - case yqlplusParser.RULE_output_statement: - Source_statementContext source_statement = statementContext.output_statement().source_statement(); - OperatorNode<SequenceOperator> query; - if (source_statement.getChildCount() == 1) { - query = convertQuery( source_statement.query_statement().getChild(0), scope); - } else { - query = convertQuery(source_statement, scope); - } - String variable = "result" + (++output); - boolean isCountVariable = false; - OperatorNode<ExpressionOperator> pageSize = null; - ParseTree outputStatement = node.getChild(0); - Location location = toLocation(scope, outputStatement); - for (int i = 1; i < outputStatement.getChildCount(); ++i) { - ParseTree child = outputStatement.getChild(i); - switch (getParseTreeIndex(child)) { - case yqlplusParser.RULE_paged_clause: - Paged_clauseContext pagedContext = (Paged_clauseContext) child; - pageSize = convertExpr(pagedContext.fixed_or_parameter(), scope); - break; - case yqlplusParser.RULE_output_spec: - Output_specContext outputSpecContext = (Output_specContext) child; - variable = outputSpecContext.ident().getText(); - if (outputSpecContext.COUNT() != null) { - isCountVariable = true; - } - break; - default: - throw new ProgramCompileException( "Unknown statement attribute: " + child.toStringTree()); - } - } - scope.defineVariable(location, variable); - if (pageSize != null) { - query = OperatorNode.create(SequenceOperator.PAGE, query, pageSize); - } - stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable)); - stmts.add(OperatorNode.create(location, isCountVariable ? StatementOperator.COUNT:StatementOperator.OUTPUT, variable)); - } - break; - } - default: - throw new ProgramCompileException("Unknown program element: " + node.getText()); } + scope.defineVariable(location, variable); + stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable)); + stmts.add(OperatorNode.create(location, isCountVariable ? StatementOperator.COUNT:StatementOperator.OUTPUT, variable)); } - // traverse the tree, find all of the namespaced calls not covered by - // imports so we can - // define "implicit" import statements for them (to make engine - // implementation easier) return OperatorNode.create(StatementOperator.PROGRAM, stmts); } @@ -982,19 +611,19 @@ final class ProgramParser { private ProjectionBuilder readProjection(List<Field_defContext> fieldDefs, Scope scope) { if (null == fieldDefs) - throw new ProgramCompileException("Null fieldDefs"); + throw new ProgramCompileException("Null fieldDefs"); ProjectionBuilder proj = new ProjectionBuilder(); for (Field_defContext rulenode : fieldDefs) { // FIELD - // expression alias_def? - OperatorNode<ExpressionOperator> expr = convertExpr((ExpressionContext)rulenode.getChild(0), scope); + // expression alias_def? + OperatorNode<ExpressionOperator> expr = convertExpr(rulenode.getChild(0), scope); - String aliasName = null; - if (rulenode.getChildCount() > 1) { - // ^(ALIAS ID) - aliasName = rulenode.alias_def().ID().getText(); - } - proj.addField(aliasName, expr); + String aliasName = null; + if (rulenode.getChildCount() > 1) { + // ^(ALIAS ID) + aliasName = rulenode.alias_def().ID().getText(); + } + proj.addField(aliasName, expr); // no grammar for the other rule types at this time } return proj; @@ -1009,359 +638,345 @@ final class ProgramParser { } public OperatorNode<ExpressionOperator> convertExpr(ParseTree parseTree, Scope scope) { - switch (getParseTreeIndex(parseTree)) { - case yqlplusParser.RULE_vespa_grouping: { - ParseTree firstChild = parseTree.getChild(0); - if (getParseTreeIndex(firstChild) == yqlplusParser.RULE_annotation) { - ParseTree secondChild = parseTree.getChild(1); - OperatorNode<ExpressionOperator> annotation = convertExpr(((AnnotationContext) firstChild) - .constantMapExpression(), scope); - OperatorNode<ExpressionOperator> expr = OperatorNode.create(toLocation(scope, secondChild), - ExpressionOperator.VESPA_GROUPING, secondChild.getText()); - List<String> names = (List<String>) annotation.getArgument(0); - List<OperatorNode<ExpressionOperator>> annotates = (List<OperatorNode<ExpressionOperator>>) annotation - .getArgument(1); - for (int i = 0; i < names.size(); ++i) { - expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); - } - return expr; - } else { - return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VESPA_GROUPING, - firstChild.getText()); - } - } - case yqlplusParser.RULE_nullOperator: - return OperatorNode.create(ExpressionOperator.NULL); - case yqlplusParser.RULE_argument: - return convertExpr(parseTree.getChild(0), scope); - case yqlplusParser.RULE_fixed_or_parameter: { - ParseTree firstChild = parseTree.getChild(0); - if (getParseTreeIndex(firstChild) == yqlplusParser.INT) { - return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.LITERAL, Integer.valueOf(firstChild.getText())); - } else { - return convertExpr(firstChild, scope); - } - } - case yqlplusParser.RULE_constantMapExpression: { - List<ConstantPropertyNameAndValueContext> propertyList = ((ConstantMapExpressionContext) parseTree).constantPropertyNameAndValue(); - List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size()); - List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithExpectedSize(propertyList.size()); - for (ConstantPropertyNameAndValueContext child : propertyList) { - // : propertyName ':' expression[$expression::namespace] -> - // ^(PROPERTY propertyName expression) - names.add(StringUnescaper.unquote(child.getChild(0).getText())); - exprs.add(convertExpr(child.getChild(2), scope)); + switch (getParseTreeIndex(parseTree)) { + case yqlplusParser.RULE_vespa_grouping: { + ParseTree firstChild = parseTree.getChild(0); + if (getParseTreeIndex(firstChild) == yqlplusParser.RULE_annotation) { + ParseTree secondChild = parseTree.getChild(1); + OperatorNode<ExpressionOperator> annotation = convertExpr(((AnnotationContext) firstChild) + .constantMapExpression(), scope); + OperatorNode<ExpressionOperator> expr = OperatorNode.create(toLocation(scope, secondChild), + ExpressionOperator.VESPA_GROUPING, secondChild.getText()); + List<String> names = annotation.getArgument(0); + List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1); + for (int i = 0; i < names.size(); ++i) { + expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); + } + return expr; + } else { + return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VESPA_GROUPING, + firstChild.getText()); + } } - return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs); - } - case yqlplusParser.RULE_mapExpression: { - List<PropertyNameAndValueContext> propertyList = ((MapExpressionContext)parseTree).propertyNameAndValue(); - List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size()); - List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithCapacity(propertyList.size()); - for (PropertyNameAndValueContext child : propertyList) { - // : propertyName ':' expression[$expression::namespace] -> - // ^(PROPERTY propertyName expression) - names.add(StringUnescaper.unquote(child.getChild(0).getText())); - exprs.add(convertExpr(child.getChild(2), scope)); - } - return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs); - } - case yqlplusParser.RULE_constantArray: { - List<ConstantExpressionContext> expressionList = ((ConstantArrayContext)parseTree).constantExpression(); - List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size()); - for (ConstantExpressionContext expr : expressionList) { - values.add(convertExpr(expr, scope)); + case yqlplusParser.RULE_nullOperator: + return OperatorNode.create(ExpressionOperator.NULL); + case yqlplusParser.RULE_argument: + return convertExpr(parseTree.getChild(0), scope); + case yqlplusParser.RULE_fixed_or_parameter: { + ParseTree firstChild = parseTree.getChild(0); + if (getParseTreeIndex(firstChild) == yqlplusParser.INT) { + return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.LITERAL, Integer.valueOf(firstChild.getText())); + } else { + return convertExpr(firstChild, scope); + } + } + case yqlplusParser.RULE_constantMapExpression: { + List<ConstantPropertyNameAndValueContext> propertyList = ((ConstantMapExpressionContext) parseTree).constantPropertyNameAndValue(); + List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size()); + List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithExpectedSize(propertyList.size()); + for (ConstantPropertyNameAndValueContext child : propertyList) { + // : propertyName ':' expression[$expression::namespace] -> + // ^(PROPERTY propertyName expression) + names.add(StringUnescaper.unquote(child.getChild(0).getText())); + exprs.add(convertExpr(child.getChild(2), scope)); + } + return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs); + } + case yqlplusParser.RULE_mapExpression: { + List<PropertyNameAndValueContext> propertyList = ((MapExpressionContext)parseTree).propertyNameAndValue(); + List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size()); + List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithCapacity(propertyList.size()); + for (PropertyNameAndValueContext child : propertyList) { + // : propertyName ':' expression[$expression::namespace] -> + // ^(PROPERTY propertyName expression) + names.add(StringUnescaper.unquote(child.getChild(0).getText())); + exprs.add(convertExpr(child.getChild(2), scope)); + } + return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs); + } + case yqlplusParser.RULE_constantArray: { + List<ConstantExpressionContext> expressionList = ((ConstantArrayContext)parseTree).constantExpression(); + List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size()); + for (ConstantExpressionContext expr : expressionList) { + values.add(convertExpr(expr, scope)); + } + return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values); + } + case yqlplusParser.RULE_arrayLiteral: { + List<ExpressionContext> expressionList = ((ArrayLiteralContext) parseTree).expression(); + List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size()); + for (ExpressionContext expr : expressionList) { + values.add(convertExpr(expr, scope)); + } + return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values); + } + // dereferencedExpression: primaryExpression(indexref[in_select]| propertyref)* + case yqlplusParser.RULE_dereferencedExpression: { + DereferencedExpressionContext dereferencedExpression = (DereferencedExpressionContext) parseTree; + Iterator<ParseTree> it = dereferencedExpression.children.iterator(); + OperatorNode<ExpressionOperator> result = convertExpr(it.next(), scope); + while (it.hasNext()) { + ParseTree defTree = it.next(); + if (getParseTreeIndex(defTree) == yqlplusParser.RULE_propertyref) { + // DOT nm=ID + result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.PROPREF, result, defTree.getChild(1).getText()); + } else { + // indexref + result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.INDEX, result, convertExpr(defTree.getChild(1), scope)); + } + } + return result; } - return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values); + case yqlplusParser.RULE_primaryExpression: { + // ^(CALL namespaced_name arguments) + ParseTree firstChild = parseTree.getChild(0); + switch (getParseTreeIndex(firstChild)) { + case yqlplusParser.RULE_fieldref: { + return convertExpr(firstChild, scope); + } + case yqlplusParser.RULE_callExpresion: { + List<ArgumentContext> args = ((ArgumentsContext) firstChild.getChild(1)).argument(); + List<OperatorNode<ExpressionOperator>> arguments = Lists.newArrayListWithExpectedSize(args.size()); + for (ArgumentContext argContext : args) { + arguments.add(convertExpr(argContext.expression(),scope)); + } + return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.CALL, scope.resolvePath(readName((Namespaced_nameContext) firstChild.getChild(0))), arguments); + } + case yqlplusParser.RULE_parameter: + // external variable reference + return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VARREF, firstChild.getChild(1).getText()); + case yqlplusParser.RULE_scalar_literal: + case yqlplusParser.RULE_arrayLiteral: + case yqlplusParser.RULE_mapExpression: + return convertExpr(firstChild, scope); + case yqlplusParser.LPAREN: + return convertExpr(parseTree.getChild(1), scope); + } + break; + } + case yqlplusParser.RULE_parameter: { + // external variable reference + ParserRuleContext parameterContext = (ParserRuleContext) parseTree; + IdentContext identContext = parameterContext.getRuleContext(IdentContext.class, 0); + return OperatorNode.create(toLocation(scope, identContext), ExpressionOperator.VARREF, identContext.getText()); + } + case yqlplusParser.RULE_annotateExpression: { + //annotation logicalORExpression + AnnotationContext annotateExpressionContext = ((AnnotateExpressionContext)parseTree).annotation(); + OperatorNode<ExpressionOperator> annotation = convertExpr(annotateExpressionContext.constantMapExpression(), scope); + OperatorNode<ExpressionOperator> expr = convertExpr(parseTree.getChild(1), scope); + List<String> names = annotation.getArgument(0); + List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1); + for (int i = 0; i < names.size(); ++i) { + expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); + } + return expr; + } + case yqlplusParser.RULE_expression: { + return convertExpr(parseTree.getChild(0), scope); + } + case yqlplusParser.RULE_logicalANDExpression: + LogicalANDExpressionContext andExpressionContext = (LogicalANDExpressionContext) parseTree; + return readConjOp(ExpressionOperator.AND, andExpressionContext.equalityExpression(), scope); + case yqlplusParser.RULE_logicalORExpression: { + int childCount = parseTree.getChildCount(); + LogicalORExpressionContext logicalORExpressionContext = (LogicalORExpressionContext) parseTree; + if (childCount > 1) { + return readConjOrOp(ExpressionOperator.OR, logicalORExpressionContext, scope); + } else { + List<EqualityExpressionContext> equalityExpressionList = ((LogicalANDExpressionContext) parseTree.getChild(0)).equalityExpression(); + if (equalityExpressionList.size() > 1) { + return readConjOp(ExpressionOperator.AND, equalityExpressionList, scope); + } else { + return convertExpr(equalityExpressionList.get(0), scope); + } + } + } + case yqlplusParser.RULE_equalityExpression: { + EqualityExpressionContext equalityExpression = (EqualityExpressionContext) parseTree; + RelationalExpressionContext relationalExpressionContext = equalityExpression.relationalExpression(0); + OperatorNode<ExpressionOperator> expr = convertExpr(relationalExpressionContext, scope); + InNotInTargetContext inNotInTarget = equalityExpression.inNotInTarget(); + int childCount = equalityExpression.getChildCount(); + if (childCount == 1) { + return expr; + } + if (inNotInTarget != null) { + Literal_listContext literalListContext = inNotInTarget.literal_list(); + boolean isIN = equalityExpression.IN() != null; + if (literalListContext == null) { + Select_statementContext selectStatementContext = inNotInTarget.select_statement(); + OperatorNode<SequenceOperator> query = convertQuery(selectStatementContext, scope); + return OperatorNode.create(expr.getLocation(),isIN ? ExpressionOperator.IN_QUERY: ExpressionOperator.NOT_IN_QUERY, expr, query); + } else { + // we need to identify the type of the target; if it's a + // scalar we need to wrap it in a CREATE_ARRAY + // if it's already a CREATE ARRAY then it's fine, otherwise + // we need to know the variable type + // return readBinOp(node.getType() == yqlplusParser.IN ? + // ExpressionOperator.IN : ExpressionOperator.NOT_IN, node, + // scope); + return readBinOp(isIN ? ExpressionOperator.IN: ExpressionOperator.NOT_IN, equalityExpression.getChild(0), literalListContext, scope); + } + + } else { + ParseTree firstChild = equalityExpression.getChild(1); + if (equalityExpression.getChildCount() == 2) { + switch (getParseTreeIndex(firstChild)) { + case yqlplusParser.IS_NULL: + return readUnOp(ExpressionOperator.IS_NULL, relationalExpressionContext, scope); + case yqlplusParser.IS_NOT_NULL: + return readUnOp(ExpressionOperator.IS_NOT_NULL, relationalExpressionContext, scope); + } + } else { + switch (getParseTreeIndex(firstChild.getChild(0))) { + case yqlplusParser.EQ: + return readBinOp(ExpressionOperator.EQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.NEQ: + return readBinOp(ExpressionOperator.NEQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.LIKE: + return readBinOp(ExpressionOperator.LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.NOTLIKE: + return readBinOp(ExpressionOperator.NOT_LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.MATCHES: + return readBinOp(ExpressionOperator.MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.NOTMATCHES: + return readBinOp(ExpressionOperator.NOT_MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.CONTAINS: + return readBinOp(ExpressionOperator.CONTAINS, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + } + } + + } + break; + } + case yqlplusParser.RULE_relationalExpression: { + RelationalExpressionContext relationalExpressionContext = (RelationalExpressionContext) parseTree; + RelationalOpContext opContext = relationalExpressionContext.relationalOp(); + if (opContext != null) { + switch (getParseTreeIndex(relationalExpressionContext.relationalOp().getChild(0))) { + case yqlplusParser.LT: + return readBinOp(ExpressionOperator.LT, parseTree, scope); + case yqlplusParser.LTEQ: + return readBinOp(ExpressionOperator.LTEQ, parseTree, scope); + case yqlplusParser.GT: + return readBinOp(ExpressionOperator.GT, parseTree, scope); + case yqlplusParser.GTEQ: + return readBinOp(ExpressionOperator.GTEQ, parseTree, scope); + } + } else { + return convertExpr(relationalExpressionContext.additiveExpression(0), scope); + } + } + break; + case yqlplusParser.RULE_additiveExpression: + case yqlplusParser.RULE_multiplicativeExpression: { + if (parseTree.getChildCount() > 1) { + String opStr = parseTree.getChild(1).getText(); + switch (opStr) { + case "+": + return readBinOp(ExpressionOperator.ADD, parseTree, scope); + case "-": + return readBinOp(ExpressionOperator.SUB, parseTree, scope); + case "/": + return readBinOp(ExpressionOperator.DIV, parseTree, scope); + case "*": + return readBinOp(ExpressionOperator.MULT, parseTree, scope); + case "%": + return readBinOp(ExpressionOperator.MOD, parseTree, scope); + default: + if (parseTree.getChild(0) instanceof UnaryExpressionContext) { + return convertExpr(parseTree.getChild(0), scope); + } else { + throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.toStringTree()); + } + } + } else { + if (parseTree.getChild(0) instanceof UnaryExpressionContext) { + return convertExpr(parseTree.getChild(0), scope); + } else if (parseTree.getChild(0) instanceof MultiplicativeExpressionContext) { + return convertExpr(parseTree.getChild(0), scope); + } else { + throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.getText()); + } + } + } + case yqlplusParser.RULE_unaryExpression: { + if (1 == parseTree.getChildCount()) { + return convertExpr(parseTree.getChild(0), scope); + } else if (2 == parseTree.getChildCount()) { + if ("-".equals(parseTree.getChild(0).getText())) { + return readUnOp(ExpressionOperator.NEGATE, parseTree, scope); + } else if ("!".equals(parseTree.getChild(0).getText())) { + return readUnOp(ExpressionOperator.NOT, parseTree, scope); + } + throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown unary operator " + parseTree.getText()); + } else { + throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown child count " + parseTree.getChildCount() + " of " + parseTree.getText()); + } + } + case yqlplusParser.RULE_fieldref: { + // all in-scope data sources should be defined in scope + // the 'first' field in a namespaced reference must be: + // - a field name if (and only if) there is exactly one data source + // in scope OR + // - an alias name, which will be followed by a field name + // ^(FIELDREF<FieldReference>[$expression::namespace] + // namespaced_name) + List<String> path = readName((Namespaced_nameContext) parseTree.getChild(0)); + Location loc = toLocation(scope, parseTree.getChild(0)); + String alias = path.get(0); + OperatorNode<ExpressionOperator> result = null; + int start = 0; + if (scope.isCursor(alias)) { + if (path.size() > 1) { + result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(1)); + start = 2; + } else { + result = OperatorNode.create(loc, ExpressionOperator.READ_RECORD, alias); + start = 1; + } + } else if (scope.isBound(alias)) { + return OperatorNode.create(loc, ExpressionOperator.READ_MODULE, scope.getBinding(alias).toPathWith(path.subList(1, path.size()))); + } else if (scope.getCursors().size() == 1) { + alias = scope.getCursors().iterator().next(); + result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(0)); + start = 1; + } else { + // ah ha, we can't end up with a 'loose' UDF call because it + // won't be a module or known alias + // so we need not support implicit imports for constants used in + // UDFs + throw new ProgramCompileException(loc, "Unknown field or alias '%s'", alias); + } + for (int idx = start; idx < path.size(); ++idx) { + result = OperatorNode.create(loc, ExpressionOperator.PROPREF, result, path.get(idx)); + } + return result; + } + case yqlplusParser.RULE_scalar_literal: + return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.LITERAL, convertLiteral((Scalar_literalContext) parseTree)); + case yqlplusParser.RULE_constantExpression: + return convertExpr(parseTree.getChild(0), scope); + case yqlplusParser.RULE_literal_list: + if (getParseTreeIndex(parseTree.getChild(1)) == yqlplusParser.RULE_array_parameter) { + return convertExpr(parseTree.getChild(1), scope); + } else { + List<Literal_elementContext> elements = ((Literal_listContext) parseTree).literal_element(); + ParseTree firldElement = elements.get(0).getChild(0); + if (elements.size() == 1 && scope.getParser().isArrayParameter(firldElement)) { + return convertExpr(firldElement, scope); + } else { + List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(elements.size()); + for (Literal_elementContext child : elements) { + values.add(convertExpr(child.getChild(0), scope)); + } + return OperatorNode.create(toLocation(scope, elements.get(0)),ExpressionOperator.ARRAY, values); + } + } } - case yqlplusParser.RULE_arrayLiteral: { - List<ExpressionContext> expressionList = ((ArrayLiteralContext) parseTree).expression(); - List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size()); - for (ExpressionContext expr : expressionList) { - values.add(convertExpr(expr, scope)); - } - return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values); - } - //dereferencedExpression: primaryExpression(indexref[in_select]| propertyref)* - case yqlplusParser.RULE_dereferencedExpression: { - DereferencedExpressionContext dereferencedExpression = (DereferencedExpressionContext) parseTree; - Iterator<ParseTree> it = dereferencedExpression.children.iterator(); - OperatorNode<ExpressionOperator> result = convertExpr(it.next(), scope); - while (it.hasNext()) { - ParseTree defTree = it.next(); - if (getParseTreeIndex(defTree) == yqlplusParser.RULE_propertyref) { - //DOT nm=ID - result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.PROPREF, result, defTree.getChild(1).getText()); - } else { - //indexref - result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.INDEX, result, convertExpr(defTree.getChild(1), scope)); - } - } - return result; - } - case yqlplusParser.RULE_primaryExpression: { - // ^(CALL namespaced_name arguments) - ParseTree firstChild = parseTree.getChild(0); - switch (getParseTreeIndex(firstChild)) { - case yqlplusParser.RULE_fieldref: { - return convertExpr(firstChild, scope); - } - case yqlplusParser.RULE_callExpresion: { - List<ArgumentContext> args = ((ArgumentsContext) firstChild.getChild(1)).argument(); - List<OperatorNode<ExpressionOperator>> arguments = Lists.newArrayListWithExpectedSize(args.size()); - for (ArgumentContext argContext : args) { - arguments.add(convertExpr(argContext.expression(),scope)); - } - return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.CALL, scope.resolvePath(readName((Namespaced_nameContext) firstChild.getChild(0))), arguments); - } - // TODO add processing this is not implemented in V3 - // case yqlplusParser.APPLY: - - case yqlplusParser.RULE_parameter: - // external variable reference - return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VARREF, firstChild.getChild(1).getText()); - case yqlplusParser.RULE_scalar_literal: - case yqlplusParser.RULE_arrayLiteral: - case yqlplusParser.RULE_mapExpression: - return convertExpr(firstChild, scope); - case yqlplusParser.LPAREN: - return convertExpr(parseTree.getChild(1), scope); - } - break; - } - - // TODO: Temporarily disable CAST - think through how types are named - // case yqlplusParser.CAST: { - // - // return new Cast() - // } - // return new CastExpression(payload); - case yqlplusParser.RULE_parameter: { - // external variable reference - ParserRuleContext parameterContext = (ParserRuleContext) parseTree; - IdentContext identContext = parameterContext.getRuleContext(IdentContext.class, 0); - return OperatorNode.create(toLocation(scope, identContext), ExpressionOperator.VARREF, identContext.getText()); - } - case yqlplusParser.RULE_annotateExpression: { - //annotation logicalORExpression - AnnotationContext annotateExpressionContext = ((AnnotateExpressionContext)parseTree).annotation(); - OperatorNode<ExpressionOperator> annotation = convertExpr(annotateExpressionContext.constantMapExpression(), scope); - OperatorNode<ExpressionOperator> expr = convertExpr(parseTree.getChild(1), scope); - List<String> names = (List<String>) annotation.getArgument(0); - List<OperatorNode<ExpressionOperator>> annotates = (List<OperatorNode<ExpressionOperator>>) annotation.getArgument(1); - for (int i = 0; i < names.size(); ++i) { - expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); - } - return expr; - } - case yqlplusParser.RULE_expression: { - return convertExpr(parseTree.getChild(0), scope); - } - case yqlplusParser.RULE_logicalANDExpression: - LogicalANDExpressionContext andExpressionContext = (LogicalANDExpressionContext) parseTree; - return readConjOp(ExpressionOperator.AND, andExpressionContext.equalityExpression(), scope); - case yqlplusParser.RULE_logicalORExpression: { - int childCount = parseTree.getChildCount(); - LogicalORExpressionContext logicalORExpressionContext = (LogicalORExpressionContext) parseTree; - if (childCount > 1) { - return readConjOrOp(ExpressionOperator.OR, logicalORExpressionContext, scope); - } else { - List<EqualityExpressionContext> equalityExpressionList = ((LogicalANDExpressionContext) parseTree.getChild(0)).equalityExpression(); - if (equalityExpressionList.size() > 1) { - return readConjOp(ExpressionOperator.AND, equalityExpressionList, scope); - } else { - return convertExpr(equalityExpressionList.get(0), scope); - } - } - } - case yqlplusParser.RULE_equalityExpression: { - EqualityExpressionContext equalityExpression = (EqualityExpressionContext) parseTree; - RelationalExpressionContext relationalExpressionContext = equalityExpression.relationalExpression(0); - OperatorNode<ExpressionOperator> expr = convertExpr(relationalExpressionContext, scope); - InNotInTargetContext inNotInTarget = equalityExpression.inNotInTarget(); - int childCount = equalityExpression.getChildCount(); - if (childCount == 1) { - return expr; - } - if (inNotInTarget != null) { - Literal_listContext literalListContext = inNotInTarget.literal_list(); - boolean isIN = equalityExpression.IN() != null; - if (literalListContext == null) { - Select_statementContext selectStatementContext = inNotInTarget.select_statement(); - OperatorNode<SequenceOperator> query = convertQuery(selectStatementContext, scope); - return OperatorNode.create(expr.getLocation(),isIN ? ExpressionOperator.IN_QUERY: ExpressionOperator.NOT_IN_QUERY, expr, query); - } else { - // we need to identify the type of the target; if it's a - // scalar we need to wrap it in a CREATE_ARRAY - // if it's already a CREATE ARRAY then it's fine, otherwise - // we need to know the variable type - // return readBinOp(node.getType() == yqlplusParser.IN ? - // ExpressionOperator.IN : ExpressionOperator.NOT_IN, node, - // scope); - return readBinOp(isIN ? ExpressionOperator.IN: ExpressionOperator.NOT_IN, equalityExpression.getChild(0), literalListContext, scope); - } - - } else { - ParseTree firstChild = equalityExpression.getChild(1); - if (equalityExpression.getChildCount() == 2) { - switch (getParseTreeIndex(firstChild)) { - case yqlplusParser.IS_NULL: - return readUnOp(ExpressionOperator.IS_NULL, relationalExpressionContext, scope); - case yqlplusParser.IS_NOT_NULL: - return readUnOp(ExpressionOperator.IS_NOT_NULL, relationalExpressionContext, scope); - } - } else { - switch (getParseTreeIndex(firstChild.getChild(0))) { - case yqlplusParser.EQ: - return readBinOp(ExpressionOperator.EQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.NEQ: - return readBinOp(ExpressionOperator.NEQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.LIKE: - return readBinOp(ExpressionOperator.LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.NOTLIKE: - return readBinOp(ExpressionOperator.NOT_LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.MATCHES: - return readBinOp(ExpressionOperator.MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.NOTMATCHES: - return readBinOp(ExpressionOperator.NOT_MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.CONTAINS: - return readBinOp(ExpressionOperator.CONTAINS, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - } - } - - } - break; - } - case yqlplusParser.RULE_relationalExpression: { - RelationalExpressionContext relationalExpressionContext = (RelationalExpressionContext) parseTree; - RelationalOpContext opContext = relationalExpressionContext.relationalOp(); - if (opContext != null) { - switch (getParseTreeIndex(relationalExpressionContext.relationalOp().getChild(0))) { - case yqlplusParser.LT: - return readBinOp(ExpressionOperator.LT, parseTree, scope); - case yqlplusParser.LTEQ: - return readBinOp(ExpressionOperator.LTEQ, parseTree, scope); - case yqlplusParser.GT: - return readBinOp(ExpressionOperator.GT, parseTree, scope); - case yqlplusParser.GTEQ: - return readBinOp(ExpressionOperator.GTEQ, parseTree, scope); - } - } else { - return convertExpr(relationalExpressionContext.additiveExpression(0), scope); - } - } - break; - case yqlplusParser.RULE_additiveExpression: - case yqlplusParser.RULE_multiplicativeExpression: { - if (parseTree.getChildCount() > 1) { - String opStr = parseTree.getChild(1).getText(); - switch (opStr) { - case "+": - return readBinOp(ExpressionOperator.ADD, parseTree, scope); - case "-": - return readBinOp(ExpressionOperator.SUB, parseTree, scope); - case "/": - return readBinOp(ExpressionOperator.DIV, parseTree, scope); - case "*": - return readBinOp(ExpressionOperator.MULT, parseTree, scope); - case "%": - return readBinOp(ExpressionOperator.MOD, parseTree, scope); - default: - if (parseTree.getChild(0) instanceof UnaryExpressionContext) { - return convertExpr(parseTree.getChild(0), scope); - } else { - throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.toStringTree()); - } - } - } else { - if (parseTree.getChild(0) instanceof UnaryExpressionContext) { - return convertExpr(parseTree.getChild(0), scope); - } else if (parseTree.getChild(0) instanceof MultiplicativeExpressionContext) { - return convertExpr(parseTree.getChild(0), scope); - } else { - throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.getText()); - } - } - } - case yqlplusParser.RULE_unaryExpression: { - if (1 == parseTree.getChildCount()) { - return convertExpr(parseTree.getChild(0), scope); - } else if (2 == parseTree.getChildCount()) { - if ("-".equals(parseTree.getChild(0).getText())) { - return readUnOp(ExpressionOperator.NEGATE, parseTree, scope); - } else if ("!".equals(parseTree.getChild(0).getText())) { - return readUnOp(ExpressionOperator.NOT, parseTree, scope); - } - throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown unary operator " + parseTree.getText()); - } else { - throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown child count " + parseTree.getChildCount() + " of " + parseTree.getText()); - } - } - case yqlplusParser.RULE_fieldref: - case yqlplusParser.RULE_joinDereferencedExpression: { - // all in-scope data sources should be defined in scope - // the 'first' field in a namespaced reference must be: - // - a field name if (and only if) there is exactly one data source - // in scope OR - // - an alias name, which will be followed by a field name - // ^(FIELDREF<FieldReference>[$expression::namespace] - // namespaced_name) - List<String> path = readName((Namespaced_nameContext) parseTree.getChild(0)); - Location loc = toLocation(scope, parseTree.getChild(0)); - String alias = path.get(0); - OperatorNode<ExpressionOperator> result = null; - int start = 0; - if (scope.isCursor(alias)) { - if (path.size() > 1) { - result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(1)); - start = 2; - } else { - result = OperatorNode.create(loc, ExpressionOperator.READ_RECORD, alias); - start = 1; - } - } else if (scope.isBound(alias)) { - return OperatorNode.create(loc, ExpressionOperator.READ_MODULE, scope.getBinding(alias).toPathWith(path.subList(1, path.size()))); - } else if (scope.getCursors().size() == 1) { - alias = scope.getCursors().iterator().next(); - result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(0)); - start = 1; - } else { - // ah ha, we can't end up with a 'loose' UDF call because it - // won't be a module or known alias - // so we need not support implicit imports for constants used in - // UDFs - throw new ProgramCompileException(loc, "Unknown field or alias '%s'", alias); - } - for (int idx = start; idx < path.size(); ++idx) { - result = OperatorNode.create(loc, ExpressionOperator.PROPREF, result, path.get(idx)); - } - return result; - } - case yqlplusParser.RULE_scalar_literal: - return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.LITERAL, convertLiteral((Scalar_literalContext) parseTree)); - case yqlplusParser.RULE_insert_values: - return readValues((Insert_valuesContext) parseTree, scope); - case yqlplusParser.RULE_constantExpression: - return convertExpr(parseTree.getChild(0), scope); - case yqlplusParser.RULE_literal_list: - if (getParseTreeIndex(parseTree.getChild(1)) == yqlplusParser.RULE_array_parameter) { - return convertExpr(parseTree.getChild(1), scope); - } else { - List<Literal_elementContext> elements = ((Literal_listContext) parseTree).literal_element(); - ParseTree firldElement = elements.get(0).getChild(0); - if (elements.size() == 1 && scope.getParser().isArrayParameter(firldElement)) { - return convertExpr(firldElement, scope); - } else { - List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(elements.size()); - for (Literal_elementContext child : elements) { - values.add(convertExpr(child.getChild(0), scope)); - } - return OperatorNode.create(toLocation(scope, elements.get(0)),ExpressionOperator.ARRAY, values); - } - } - } - throw new ProgramCompileException(toLocation(scope, parseTree), - "Unknown expression type: " + parseTree.getText()); + throw new ProgramCompileException(toLocation(scope, parseTree), + "Unknown expression type: " + parseTree.getText()); } public Object convertLiteral(Scalar_literalContext literal) { @@ -1375,9 +990,9 @@ final class ProgramParser { case yqlplusParser.STRING: return StringUnescaper.unquote(text); case yqlplusParser.TRUE: - return Boolean.valueOf(true); + return true; case yqlplusParser.FALSE: - return Boolean.valueOf(false); + return false; case yqlplusParser.LONG_INT: return Long.parseLong(text.substring(0, text.length()-1)); default: @@ -1391,21 +1006,24 @@ final class ProgramParser { return node.getArgument(0); case MAP: { ImmutableMap.Builder<String, Object> map = ImmutableMap.builder(); - List<String> names = (List<String>) node.getArgument(0); - List<OperatorNode<ExpressionOperator>> exprs = (List<OperatorNode<ExpressionOperator>>) node.getArgument(1); + List<String> names = node.getArgument(0); + List<OperatorNode<ExpressionOperator>> exprs = node.getArgument(1); for (int i = 0; i < names.size(); ++i) { map.put(names.get(i), readConstantExpression(exprs.get(i))); } return map.build(); } case ARRAY: { - List<OperatorNode<ExpressionOperator>> exprs = (List<OperatorNode<ExpressionOperator>>) node.getArgument(0); + List<OperatorNode<ExpressionOperator>> exprs = node.getArgument(0); ImmutableList.Builder<Object> lst = ImmutableList.builder(); for (OperatorNode<ExpressionOperator> expr : exprs) { lst.add(readConstantExpression(expr)); } return lst.build(); } + case VARREF: { + return node; // must be dereferenced in YqlParser when we have userQuery + } default: throw new ProgramCompileException(node.getLocation(), "Internal error: Unknown constant expression type: " + node.getOperator()); } @@ -1460,77 +1078,7 @@ final class ProgramParser { } } - private OperatorNode<ExpressionOperator> readValues(Field_names_specContext nameDefs, Field_values_specContext values, Scope scope) { - List<Field_defContext> fieldDefs = nameDefs.field_def(); - List<ExpressionContext> valueDefs = values.expression(); - assert fieldDefs.size() == valueDefs.size(); - List<String> fieldNames; - List<OperatorNode<ExpressionOperator>> fieldValues; - int numPairs = fieldDefs.size(); - fieldNames = Lists.newArrayListWithExpectedSize(numPairs); - fieldValues = Lists.newArrayListWithExpectedSize(numPairs); - for (int i = 0; i < numPairs; i++) { - fieldNames.add((String) convertExpr(fieldDefs.get(i).expression(), scope).getArgument(1)); - fieldValues.add(convertExpr(valueDefs.get(i), scope)); - } - return OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues); - } - - private OperatorNode<ExpressionOperator> readValues(ParserRuleContext node, Scope scope) { - List<String> fieldNames; - List<OperatorNode<ExpressionOperator>> fieldValues; - if (node.getRuleIndex() == yqlplusParser.RULE_field_def) { - Field_defContext fieldDefContext = (Field_defContext)node; - //TODO double check - fieldNames = Lists.newArrayListWithExpectedSize(node.getChildCount()); - fieldValues = Lists.newArrayListWithExpectedSize(node.getChildCount()); - for (int i = 0; i < node.getChildCount(); i++) { - fieldNames.add((String) convertExpr(node.getChild(i).getChild(0).getChild(0), scope).getArgument(1)); - fieldValues.add(convertExpr(node.getChild(i).getChild(0).getChild(1), scope)); - } - } else { - assert node.getChildCount() % 2 == 0; - int numPairs = node.getChildCount() / 2; - fieldNames = Lists.newArrayListWithExpectedSize(numPairs); - fieldValues = Lists.newArrayListWithExpectedSize(numPairs); - for (int i = 0; i < numPairs; i++) { - fieldNames.add((String) convertExpr(node.getChild(i).getChild(0), scope).getArgument(1)); - fieldValues.add(convertExpr(node.getChild(numPairs + i), scope)); - } - } - return OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues); - } - - /* - * Converts node list - * - * a_name, b_name, c_name, a_value_1, b_value_1, c_value_1, a_value_2, b_value_2, c_value2, a_value_3, b_value_3, c_value_3 - * - * into corresponding constant sequence: - * - * [ { a_name : a_value_1, b_name : b_value_1, c_name : c_value_1 }, ... ] - * - */ - private OperatorNode<SequenceOperator> readBatchValues(Field_names_specContext nameDefs, List<Field_values_group_specContext> valueGroups, Scope scope) { - List<Field_defContext> nameContexts = nameDefs.field_def(); - List<String> fieldNames = Lists.newArrayList(); - for (Field_defContext nameContext:nameContexts) { - fieldNames.add((String) convertExpr(nameContext.getChild(0), scope).getArgument(1)); - } - List<OperatorNode> records = Lists.newArrayList(); - for (Field_values_group_specContext valueGorup:valueGroups) { - List<ExpressionContext> expressionList = valueGorup.expression(); - List<OperatorNode<ExpressionOperator>> fieldValues = Lists.newArrayListWithExpectedSize(expressionList.size()); - for (ExpressionContext expressionContext:expressionList) { - fieldValues.add(convertExpr(expressionContext, scope)); - } - records.add(OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues)); - } - // Return constant sequence of records with the given name/values - return OperatorNode.create(SequenceOperator.EVALUATE, OperatorNode.create(ExpressionOperator.ARRAY, records)); - } - - /* + /** * Scans the given node for READ_FIELD expressions. * * TODO: Search recursively and consider additional operators @@ -1555,4 +1103,5 @@ final class ProgramParser { } return readFieldList; } + } diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 6eef1252998..9e6701572dc 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -16,6 +16,7 @@ import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; import static com.yahoo.search.yql.YqlParser.END_ANCHOR; import static com.yahoo.search.yql.YqlParser.EQUIV; import static com.yahoo.search.yql.YqlParser.FILTER; +import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; import static com.yahoo.search.yql.YqlParser.LABEL; @@ -72,6 +73,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.MarkerWordItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; @@ -140,7 +142,7 @@ public class VespaSerializer { escape(((WordItem) current).getIndexedString(), destination).append('"'); } else { throw new IllegalArgumentException("Serializing of " + current.getClass().getSimpleName() - + " in segment AND expressions not implemented, please report this as a bug."); + + " in segment AND expressions not implemented, please report this as a bug."); } } } @@ -641,7 +643,7 @@ public class VespaSerializer { WordAlternativesSerializer.serialize(destination, (WordAlternativesItem) current, false); } else { throw new IllegalArgumentException("Serializing of " + current.getClass().getSimpleName() + - " in phrases not implemented, please report this as a bug."); + " in phrases not implemented, please report this as a bug."); } } destination.append(')'); @@ -689,6 +691,26 @@ public class VespaSerializer { } + private static class GeoLocationSerializer extends Serializer<GeoLocationItem> { + @Override + void onExit(StringBuilder destination, GeoLocationItem item) { } + @Override + boolean serialize(StringBuilder destination, GeoLocationItem item) { + String annotations = leafAnnotations(item); + if (annotations.length() > 0) { + destination.append("([{").append(annotations).append("}]"); + } + destination.append(GEO_LOCATION).append('('); + destination.append(item.getIndexName()).append(", "); + var loc = item.getLocation(); + destination.append(loc.degNS()).append(", "); + destination.append(loc.degEW()).append(", "); + destination.append('"').append(loc.degRadius()).append(" deg").append('"'); + destination.append(')'); + return false; + } + } + private static class NearestNeighborSerializer extends Serializer<NearestNeighborItem> { @Override @@ -701,7 +723,24 @@ public class VespaSerializer { destination.append(leafAnnotations(item)); comma(destination, initLen); int targetNumHits = item.getTargetNumHits(); - destination.append("\"targetNumHits\": ").append(targetNumHits); + annotationKey(destination, YqlParser.TARGET_NUM_HITS).append(targetNumHits); + double distanceThreshold = item.getDistanceThreshold(); + if (distanceThreshold < Double.POSITIVE_INFINITY) { + comma(destination, initLen); + String key = YqlParser.DISTANCE_THRESHOLD; + annotationKey(destination, key).append(distanceThreshold); + } + int explore = item.getHnswExploreAdditionalHits(); + if (explore != 0) { + comma(destination, initLen); + String key = YqlParser.HNSW_EXPLORE_ADDITIONAL_HITS; + annotationKey(destination, key).append(explore); + } + boolean allow_approx = item.getAllowApproximate(); + if (! allow_approx) { + comma(destination, initLen); + annotationKey(destination, "approximate").append(allow_approx); + } destination.append("}]"); destination.append(NEAREST_NEIGHBOR).append('('); destination.append(item.getIndexName()).append(", "); @@ -911,12 +950,13 @@ public class VespaSerializer { } + @SuppressWarnings("deprecation") private static class WeakAndSerializer extends Serializer<WeakAndItem> { @Override void onExit(StringBuilder destination, WeakAndItem item) { destination.append(')'); - if (needsAnnotationBlock((WeakAndItem) item)) { + if (needsAnnotationBlock(item)) { destination.append(')'); } } @@ -1125,7 +1165,7 @@ public class VespaSerializer { Serializer doIt = dispatch.get(item.getClass()); if (doIt == null) { - throw new IllegalArgumentException(item.getClass() + " not supported for YQL+ marshalling."); + throw new IllegalArgumentException(item.getClass() + " not supported for YQL marshalling."); } if (state.peekFirst() != null && state.peekFirst().subItems > 0) { @@ -1152,6 +1192,7 @@ public class VespaSerializer { dispatchBuilder.put(EquivItem.class, new EquivSerializer()); dispatchBuilder.put(ExactStringItem.class, new WordSerializer()); dispatchBuilder.put(IntItem.class, new NumberSerializer()); + dispatchBuilder.put(GeoLocationItem.class, new GeoLocationSerializer()); dispatchBuilder.put(BoolItem.class, new BoolSerializer()); dispatchBuilder.put(MarkerWordItem.class, new WordSerializer()); // gotcha dispatchBuilder.put(NearItem.class, new NearSerializer()); @@ -1347,6 +1388,11 @@ public class VespaSerializer { } } + private static StringBuilder annotationKey(StringBuilder annotation, String val) { + annotation.append("\"").append(val).append("\": "); + return annotation; + } + private static void comma(StringBuilder annotation, int initLen) { if (annotation.length() > initLen) { annotation.append(", "); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 8d013e501e8..9eaea47ea1b 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -19,11 +19,14 @@ import com.google.common.annotations.Beta; import com.google.common.base.Preconditions; import com.yahoo.collections.LazyMap; import com.yahoo.collections.LazySet; +import com.yahoo.geo.DistanceParser; +import com.yahoo.geo.ParsedDegree; import com.yahoo.language.Language; import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.Segmenter; import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.Location; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.BoolItem; @@ -34,6 +37,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; @@ -63,6 +67,7 @@ import com.yahoo.prelude.query.WeakAndItem; import com.yahoo.prelude.query.WeightedSetItem; import com.yahoo.prelude.query.WordAlternativesItem; import com.yahoo.prelude.query.WordItem; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.grouping.Continuation; import com.yahoo.search.grouping.request.GroupingOperation; @@ -94,8 +99,8 @@ import com.yahoo.search.query.parser.ParserFactory; */ public class YqlParser implements Parser { - private static final String DESCENDING_HITS_ORDER = "descending"; - private static final String ASCENDING_HITS_ORDER = "ascending"; + public static final String DESCENDING_HITS_ORDER = "descending"; + public static final String ASCENDING_HITS_ORDER = "ascending"; private enum SegmentWhen { NEVER, POSSIBLY, ALWAYS; @@ -107,12 +112,12 @@ public class YqlParser implements Parser { private static final Integer DEFAULT_HITS = 10; private static final Integer DEFAULT_OFFSET = 0; - private static final Integer DEFAULT_TARGET_NUM_HITS = 10; + public static final Integer DEFAULT_TARGET_NUM_HITS = 10; private static final String ACCENT_DROP_DESCRIPTION = "setting for whether to remove accents if field implies it"; - private static final String ANNOTATIONS = "annotations"; + public static final String ANNOTATIONS = "annotations"; private static final String FILTER_DESCRIPTION = "term filter setting"; private static final String IMPLICIT_TRANSFORMS_DESCRIPTION = "setting for whether built-in query transformers should touch the term"; - private static final String NFKC = "nfkc"; + public static final String NFKC = "nfkc"; private static final String NORMALIZE_CASE_DESCRIPTION = "setting for whether to do case normalization if field implies it"; private static final String ORIGIN_DESCRIPTION = "string origin for a term"; private static final String RANKED_DESCRIPTION = "setting for whether to use term for ranking"; @@ -121,7 +126,7 @@ public class YqlParser implements Parser { private static final String USER_INPUT_ALLOW_EMPTY = "allowEmpty"; private static final String USER_INPUT_DEFAULT_INDEX = "defaultIndex"; private static final String USER_INPUT_GRAMMAR = "grammar"; - private static final String USER_INPUT_LANGUAGE = "language"; + public static final String USER_INPUT_LANGUAGE = "language"; private static final String USER_INPUT_RAW = "raw"; private static final String USER_INPUT_SEGMENT = "segment"; private static final String USER_INPUT = "userInput"; @@ -134,52 +139,57 @@ public class YqlParser implements Parser { public static final String SORTING_LOCALE = "locale"; public static final String SORTING_STRENGTH = "strength"; - static final String ACCENT_DROP = "accentDrop"; - static final String ALTERNATIVES = "alternatives"; - static final String AND_SEGMENTING = "andSegmenting"; - static final String BOUNDS = "bounds"; - static final String BOUNDS_LEFT_OPEN = "leftOpen"; - static final String BOUNDS_OPEN = "open"; - static final String BOUNDS_RIGHT_OPEN = "rightOpen"; - static final String CONNECTION_ID = "id"; - static final String CONNECTION_WEIGHT = "weight"; - static final String CONNECTIVITY = "connectivity"; - static final String DISTANCE = "distance"; - static final String DOT_PRODUCT = "dotProduct"; - static final String EQUIV = "equiv"; - static final String FILTER = "filter"; - static final String HIT_LIMIT = "hitLimit"; - static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; - static final String LABEL = "label"; - static final String NEAR = "near"; - static final String NEAREST_NEIGHBOR = "nearestNeighbor"; - static final String NORMALIZE_CASE = "normalizeCase"; - static final String ONEAR = "onear"; - static final String ORIGIN_LENGTH = "length"; - static final String ORIGIN_OFFSET = "offset"; - static final String ORIGIN = "origin"; - static final String ORIGIN_ORIGINAL = "original"; - static final String PHRASE = "phrase"; - static final String PREDICATE = "predicate"; - static final String PREFIX = "prefix"; - static final String RANGE = "range"; - static final String RANKED = "ranked"; - static final String RANK = "rank"; - static final String SAME_ELEMENT = "sameElement"; - static final String SCORE_THRESHOLD = "scoreThreshold"; - static final String SIGNIFICANCE = "significance"; - static final String STEM = "stem"; - static final String SUBSTRING = "substring"; - static final String SUFFIX = "suffix"; - static final String TARGET_NUM_HITS = "targetNumHits"; - static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; - static final String UNIQUE_ID = "id"; - static final String USE_POSITION_DATA = "usePositionData"; - static final String WAND = "wand"; - static final String WEAK_AND = "weakAnd"; - static final String WEIGHTED_SET = "weightedSet"; - static final String WEIGHT = "weight"; - static final String URI = "uri"; + public static final String ACCENT_DROP = "accentDrop"; + public static final String ALTERNATIVES = "alternatives"; + public static final String AND_SEGMENTING = "andSegmenting"; + public static final String APPROXIMATE = "approximate"; + public static final String BOUNDS = "bounds"; + public static final String BOUNDS_LEFT_OPEN = "leftOpen"; + public static final String BOUNDS_OPEN = "open"; + public static final String BOUNDS_RIGHT_OPEN = "rightOpen"; + public static final String CONNECTION_ID = "id"; + public static final String CONNECTION_WEIGHT = "weight"; + public static final String CONNECTIVITY = "connectivity"; + public static final String DISTANCE = "distance"; + public static final String DOT_PRODUCT = "dotProduct"; + public static final String EQUIV = "equiv"; + public static final String FILTER = "filter"; + public static final String GEO_LOCATION = "geoLocation"; + public static final String HIT_LIMIT = "hitLimit"; + public static final String DISTANCE_THRESHOLD = "distanceThreshold"; + public static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits"; + public static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; + public static final String LABEL = "label"; + public static final String NEAR = "near"; + public static final String NEAREST_NEIGHBOR = "nearestNeighbor"; + public static final String NORMALIZE_CASE = "normalizeCase"; + public static final String ONEAR = "onear"; + public static final String ORIGIN_LENGTH = "length"; + public static final String ORIGIN_OFFSET = "offset"; + public static final String ORIGIN = "origin"; + public static final String ORIGIN_ORIGINAL = "original"; + public static final String PHRASE = "phrase"; + public static final String PREDICATE = "predicate"; + public static final String PREFIX = "prefix"; + public static final String RANGE = "range"; + public static final String RANKED = "ranked"; + public static final String RANK = "rank"; + public static final String SAME_ELEMENT = "sameElement"; + public static final String SCORE_THRESHOLD = "scoreThreshold"; + public static final String SIGNIFICANCE = "significance"; + public static final String STEM = "stem"; + public static final String SUBSTRING = "substring"; + public static final String SUFFIX = "suffix"; + public static final String TARGET_HITS = "targetHits"; + public static final String TARGET_NUM_HITS = "targetNumHits"; + public static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; + public static final String UNIQUE_ID = "id"; + public static final String USE_POSITION_DATA = "usePositionData"; + public static final String WAND = "wand"; + public static final String WEAK_AND = "weakAnd"; + public static final String WEIGHTED_SET = "weightedSet"; + public static final String WEIGHT = "weight"; + public static final String URI = "uri"; private final IndexFacts indexFacts; private final List<ConnectedItem> connectedItems = new ArrayList<>(); @@ -310,10 +320,10 @@ public class YqlParser implements Parser { private void connectItems() { for (ConnectedItem entry : connectedItems) { TaggableItem to = identifiedItems.get(entry.toId); - Preconditions.checkNotNull(to, - "Item '%s' was specified to connect to item with ID %s, which does not " - + "exist in the query.", entry.fromItem, - entry.toId); + if (to == null) + throw new IllegalArgumentException("Item '" + entry.fromItem + + "' was specified to connect to item with ID " + entry.toId + + ", which does not exist in the query."); entry.fromItem.setConnectivity((Item) to, entry.weight); } } @@ -369,6 +379,8 @@ public class YqlParser implements Parser { return buildWeightedSet(ast); case DOT_PRODUCT: return buildDotProduct(ast); + case GEO_LOCATION: + return buildGeoLocation(ast); case NEAREST_NEIGHBOR: return buildNearestNeighbor(ast); case PREDICATE: @@ -410,20 +422,60 @@ public class YqlParser implements Parser { return fillWeightedSet(ast, args.get(1), new DotProductItem(getIndex(args.get(0)))); } + private Item buildGeoLocation(OperatorNode<ExpressionOperator> ast) { + List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); + Preconditions.checkArgument(args.size() == 4, "Expected 4 arguments, got %s.", args.size()); + String field = fetchFieldRead(args.get(0)); + var coord_1 = ParsedDegree.fromString(fetchFieldRead(args.get(1)), true, false); + var coord_2 = ParsedDegree.fromString(fetchFieldRead(args.get(2)), false, true); + double radius = DistanceParser.parse(fetchFieldRead(args.get(3))); + var loc = new Location(); + if (coord_1.isLatitude && coord_2.isLongitude) { + loc.setGeoCircle(coord_1.degrees, coord_2.degrees, radius); + } else if (coord_2.isLatitude && coord_1.isLongitude) { + loc.setGeoCircle(coord_2.degrees, coord_1.degrees, radius); + } else { + throw new IllegalArgumentException("Invalid geoLocation coordinates '"+coord_1+"' and '"+coord_2+"'"); + } + var item = new GeoLocationItem(loc, field); + String label = getAnnotation(ast, LABEL, String.class, null, "item label"); + if (label != null) { + item.setLabel(label); + } + return item; + } + private Item buildNearestNeighbor(OperatorNode<ExpressionOperator> ast) { List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); Preconditions.checkArgument(args.size() == 2, "Expected 2 arguments, got %s.", args.size()); String field = fetchFieldRead(args.get(0)); String property = fetchFieldRead(args.get(1)); NearestNeighborItem item = new NearestNeighborItem(field, property); - Integer targetNumHits = getAnnotation(ast, TARGET_NUM_HITS, + Integer targetNumHits = getAnnotation(ast, TARGET_HITS, + Integer.class, null, "desired minimum hits to produce"); + if (targetNumHits == null) { + targetNumHits = getAnnotation(ast, TARGET_NUM_HITS, Integer.class, null, "desired minimum hits to produce"); + } if (targetNumHits != null) { item.setTargetNumHits(targetNumHits); } + Double distanceThreshold = getAnnotation(ast, DISTANCE_THRESHOLD, + Double.class, null, "maximum distance allowed from query point"); + if (distanceThreshold != null) { + item.setDistanceThreshold(distanceThreshold); + } + Integer hnswExploreAdditionalHits = getAnnotation(ast, HNSW_EXPLORE_ADDITIONAL_HITS, + Integer.class, null, "number of extra hits to explore for HNSW algorithm"); + if (hnswExploreAdditionalHits != null) { + item.setHnswExploreAdditionalHits(hnswExploreAdditionalHits); + } + Boolean allowApproximate = getAnnotation(ast, APPROXIMATE, + Boolean.class, Boolean.TRUE, "allow approximate nearest neighbor search"); + item.setAllowApproximate(allowApproximate); String label = getAnnotation(ast, LABEL, String.class, null, "item label"); if (label != null) { - item.setLabel(label); + item.setLabel(label); } return item; } @@ -494,9 +546,13 @@ public class YqlParser implements Parser { List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); Preconditions.checkArgument(args.size() == 2, "Expected 2 arguments, got %s.", args.size()); - WandItem out = new WandItem(getIndex(args.get(0)), getAnnotation(ast, - TARGET_NUM_HITS, Integer.class, DEFAULT_TARGET_NUM_HITS, - "desired number of hits to accumulate in wand")); + Integer targetNumHits = getAnnotation(ast, TARGET_HITS, + Integer.class, null, "desired number of hits to accumulate in wand"); + if (targetNumHits == null) { + targetNumHits = getAnnotation(ast, TARGET_NUM_HITS, + Integer.class, DEFAULT_TARGET_NUM_HITS, "desired number of hits to accumulate in wand"); + } + WandItem out = new WandItem(getIndex(args.get(0)), targetNumHits); Double scoreThreshold = getAnnotation(ast, SCORE_THRESHOLD, Double.class, null, "min score for hit inclusion"); if (scoreThreshold != null) { @@ -733,7 +789,7 @@ public class YqlParser implements Parser { try { ast = new ProgramParser().parse("query", currentlyParsing.getQuery()); } catch (Exception e) { - throw new IllegalArgumentException(e); + throw new IllegalInputException(e); } assertHasOperator(ast, StatementOperator.PROGRAM); Preconditions.checkArgument(ast.getArguments().length == 1, @@ -760,10 +816,11 @@ public class YqlParser implements Parser { OperatorNode<ExpressionOperator> groupingAst = ast.<List<OperatorNode<ExpressionOperator>>> getArgument(2).get(0); GroupingOperation groupingOperation = GroupingOperation.fromString(groupingAst.<String> getArgument(0)); VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation); - List<String> continuations = getAnnotation(groupingAst, "continuations", List.class, + List<Object> continuations = getAnnotation(groupingAst, "continuations", List.class, Collections.emptyList(), "grouping continuations"); - for (String continuation : continuations) { - groupingStep.continuations().add(Continuation.fromString(continuation)); + + for (Object continuation : continuations) { + groupingStep.continuations().add(Continuation.fromString(dereference(continuation))); } groupingSteps.add(groupingStep); ast = ast.getArgument(0); @@ -772,6 +829,18 @@ public class YqlParser implements Parser { return ast; } + private String dereference(Object constantOrVarref) { + if (constantOrVarref instanceof OperatorNode) { + OperatorNode<?> varref = (OperatorNode<?>)constantOrVarref; + Preconditions.checkState(userQuery != null, + "properties must be available when trying to fetch user input"); + return userQuery.properties().getString(varref.getArgument(0, String.class)); + } + else { + return constantOrVarref.toString(); + } + } + private OperatorNode<?> fetchSorting(OperatorNode<?> ast) { if (ast.getOperator() != SequenceOperator.SORT) return ast; @@ -883,6 +952,8 @@ public class YqlParser implements Parser { private static String fetchFieldRead(OperatorNode<ExpressionOperator> ast) { switch (ast.getOperator()) { + case LITERAL: + return ast.getArgument(0).toString(); case READ_FIELD: return ast.getArgument(1); case PROPREF: @@ -953,40 +1024,52 @@ public class YqlParser implements Parser { private String fetchConditionIndex(OperatorNode<ExpressionOperator> ast) { OperatorNode<ExpressionOperator> lhs = ast.getArgument(0); OperatorNode<ExpressionOperator> rhs = ast.getArgument(1); - if (lhs.getOperator() == ExpressionOperator.LITERAL || lhs.getOperator() == ExpressionOperator.NEGATE) { + if (isNumber(lhs)) return getIndex(rhs); - } - if (rhs.getOperator() == ExpressionOperator.LITERAL || rhs.getOperator() == ExpressionOperator.NEGATE) { + else if (isNumber(rhs)) return getIndex(lhs); - } - throw new IllegalArgumentException("Expected LITERAL and READ_FIELD/PROPREF, got " + lhs.getOperator() + - " and " + rhs.getOperator() + "."); + else + throw new IllegalArgumentException("Expected LITERAL/VARREF and READ_FIELD/PROPREF, got " + lhs.getOperator() + + " and " + rhs.getOperator() + "."); + } + + private boolean isNumber(OperatorNode<ExpressionOperator> ast) { + return ast.getOperator() == ExpressionOperator.NEGATE || + ast.getOperator() == ExpressionOperator.LITERAL || ast.getOperator() == ExpressionOperator.VARREF; } - private static String getNumberAsString(OperatorNode<ExpressionOperator> ast) { + private String getNumberAsString(OperatorNode<ExpressionOperator> ast) { String negative = ""; - OperatorNode<ExpressionOperator> currentAst = ast; - if (currentAst.getOperator() == ExpressionOperator.NEGATE) { + if (ast.getOperator() == ExpressionOperator.NEGATE) { negative = "-"; - currentAst = currentAst.getArgument(0); + ast = ast.getArgument(0); + } + switch (ast.getOperator()) { + case VARREF: + Preconditions.checkState(userQuery != null, + "properties must be available when trying to fetch user input"); + return negative + userQuery.properties().getString(ast.getArgument(0, String.class)); + case LITERAL: + return negative + ast.getArgument(0).toString(); + default: + throw new IllegalArgumentException("Expected VARREF or LITERAL, got " + ast.getOperator()); } - assertHasOperator(currentAst, ExpressionOperator.LITERAL); - return negative + currentAst.getArgument(0).toString(); } - private static String fetchConditionWord(OperatorNode<ExpressionOperator> ast) { + private String fetchConditionWord(OperatorNode<ExpressionOperator> ast) { OperatorNode<ExpressionOperator> lhs = ast.getArgument(0); OperatorNode<ExpressionOperator> rhs = ast.getArgument(1); - if (lhs.getOperator() == ExpressionOperator.LITERAL || lhs.getOperator() == ExpressionOperator.NEGATE) { + if (isNumber(lhs)) { assertFieldName(rhs); return getNumberAsString(lhs); } - if (rhs.getOperator() == ExpressionOperator.LITERAL || rhs.getOperator() == ExpressionOperator.NEGATE) { + else if (isNumber(rhs)) { assertFieldName(lhs); return getNumberAsString(rhs); } - throw new IllegalArgumentException("Expected LITERAL/NEGATE and READ_FIELD/PROPREF, got " - + lhs.getOperator() + " and " + rhs.getOperator() + "."); + else + throw new IllegalArgumentException("Expected LITERAL/NEGATE and READ_FIELD/PROPREF, got " + + lhs.getOperator() + " and " + rhs.getOperator() + "."); } private static boolean isIndexOnLeftHandSide(OperatorNode<ExpressionOperator> ast) { @@ -1016,10 +1099,15 @@ public class YqlParser implements Parser { return convertVarArgs(spec, 0, new OrItem()); } + @SuppressWarnings("deprecation") private CompositeItem buildWeakAnd(OperatorNode<ExpressionOperator> spec) { WeakAndItem weakAnd = new WeakAndItem(); - Integer targetNumHits = getAnnotation(spec, TARGET_NUM_HITS, + Integer targetNumHits = getAnnotation(spec, TARGET_HITS, + Integer.class, null, "desired minimum hits to produce"); + if (targetNumHits == null) { + targetNumHits = getAnnotation(spec, TARGET_NUM_HITS, Integer.class, null, "desired minimum hits to produce"); + } if (targetNumHits != null) { weakAnd.setN(targetNumHits); } @@ -1202,7 +1290,7 @@ public class YqlParser implements Parser { equiv.setIndexName(field); for (OperatorNode<ExpressionOperator> arg : args) { switch (arg.getOperator()) { - case LITERAL: + case LITERAL: case VARREF: equiv.addItem(instantiateWordItem(field, arg, equiv.getClass())); break; case CALL: @@ -1211,7 +1299,7 @@ public class YqlParser implements Parser { break; default: throw newUnexpectedArgumentException(arg.getOperator(), - ExpressionOperator.CALL, ExpressionOperator.LITERAL); + ExpressionOperator.CALL, ExpressionOperator.LITERAL, ExpressionOperator.VARREF); } } return leafStyleSettings(ast, equiv); @@ -1272,7 +1360,8 @@ public class YqlParser implements Parser { } private Item instantiateWordItem(String field, - OperatorNode<ExpressionOperator> ast, Class<?> parent, + OperatorNode<ExpressionOperator> ast, + Class<?> parent, SegmentWhen segmentPolicy) { String wordData = getStringContents(ast); return instantiateWordItem(field, wordData, ast, parent, segmentPolicy, null, decideParsingLanguage(ast, wordData)); @@ -1288,7 +1377,8 @@ public class YqlParser implements Parser { // which always expands first, but not using getIndex, which performs checks that doesn't always work private Item instantiateWordItem(String field, String rawWord, - OperatorNode<ExpressionOperator> ast, Class<?> parent, + OperatorNode<ExpressionOperator> ast, + Class<?> parent, SegmentWhen segmentPolicy, Boolean exactMatch, Language language) { @@ -1682,7 +1772,15 @@ public class YqlParser implements Parser { Object value = ast.getAnnotation(key); for (Iterator<OperatorNode<?>> i = annotationStack.iterator(); value == null && considerParents && i.hasNext();) { - value = i.next().getAnnotation(key); + OperatorNode node = i.next(); + if (node.getOperator() == ExpressionOperator.VARREF) { + Preconditions.checkState(userQuery != null, + "properties must be available when trying to fetch user input"); + value = userQuery.properties().getString(ast.getArgument(0, String.class)); + } + else { + value = node.getAnnotation(key); + } } if (value == null) return defaultValue; Preconditions.checkArgument(expectedClass.isInstance(value), diff --git a/container-search/src/main/java/com/yahoo/text/interpretation/Annotations.java b/container-search/src/main/java/com/yahoo/text/interpretation/Annotations.java index 5afe51d4415..f625bedca19 100644 --- a/container-search/src/main/java/com/yahoo/text/interpretation/Annotations.java +++ b/container-search/src/main/java/com/yahoo/text/interpretation/Annotations.java @@ -8,28 +8,26 @@ import java.util.Map; /** * An annotation is a description of a an area of text, with a given class. For example, an annotation for the * - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + * @author Arne Bergene Fossaa */ public class Annotations { - - private Span span; + private final Span span; protected Map<String,Object> annotations; - /** * Adds an annotation to the the the set of annotations. */ public void put(String key,Object o) { - if(annotations == null) { + if (annotations == null) { annotations = new HashMap<>(); } annotations.put(key,o); } public Map<String,Object> getMap() { - if(annotations == null) { + if (annotations == null) { return Collections.emptyMap(); } else { return annotations; @@ -113,12 +111,11 @@ public class Annotations { */ public Boolean getBoolean(String key) { Object o = getMap().get(key); - if(o == null || !(o instanceof Boolean)) { + if ( ! (o instanceof Boolean)) { return null; } else { - return (Boolean) o; + return (Boolean)o; } } - } diff --git a/container-search/src/main/java/com/yahoo/text/interpretation/Interpretation.java b/container-search/src/main/java/com/yahoo/text/interpretation/Interpretation.java index 9a9b8e81633..f662a0a4e7e 100644 --- a/container-search/src/main/java/com/yahoo/text/interpretation/Interpretation.java +++ b/container-search/src/main/java/com/yahoo/text/interpretation/Interpretation.java @@ -21,21 +21,19 @@ import java.util.Set; * is not needed. * * @see Span - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + * @author Arne Bergene Fossaa */ public class Interpretation { - private Modification modification; + private final Modification modification; private double probability; - private Span rootSpan; + private final Span rootSpan; public final static AnnotationClass INTERPRETATION_CLASS = new AnnotationClass("interpretation"); - /** * Creates a new interpretation and a new modification from the text, * with the probability set to the default value(0.0). - */ public Interpretation(String text) { this(text,0.0); @@ -48,7 +46,6 @@ public class Interpretation { this(new Modification(text),probabilty); } - /** * Creates a new interpretation based on the modification, with the probability set to the default value(0.0). */ @@ -65,12 +62,10 @@ public class Interpretation { setProbability(probability); } - public Modification getModification() { return modification; } - /** * The probability that this interpretation is correct. * @return a value between 0.0 and 1.0 that gives the probability that this interpretation is correct @@ -98,13 +93,12 @@ public class Interpretation { /** Returns the root of the tree representation of the interpretation */ public Span root() { return rootSpan; } - // Wrapper methods for Span /** * Return the annotation with the given annotationclass (and create it if necessary). - * @param annotationClass The class of the annotation * + * @param annotationClass The class of the annotation */ public Annotations annotate(String annotationClass) { return annotate(new AnnotationClass(annotationClass)); @@ -112,8 +106,8 @@ public class Interpretation { /** * Return the annotation with the given annotationclass (and create it if necessary). - * @param annotationClass The class of the annotation * + * @param annotationClass The class of the annotation */ public Annotations annotate(AnnotationClass annotationClass) { return rootSpan.annotate(annotationClass); @@ -124,6 +118,7 @@ public class Interpretation { * exist, a new is created. * * A shortcut for annotate(annotationClass).put(key,value) + * * @param annotationClass class of the annotation * @param key key of the property to set on the annotation * @param value value of the property to set on the annotation @@ -137,6 +132,7 @@ public class Interpretation { * exist, a new is created. * * A shortcut for annotate(annotationClass).put(key,value) + * * @param annotationClass class of the annotation * @param key key of the property to set on the annotation * @param value value of the property to set on the annotation @@ -147,6 +143,7 @@ public class Interpretation { /** * Returns the annotation with the given annotationClass (and create it if necessary). + * * @param from start of the substring * @param to end of the substring * @param annotationClass class of the annotation @@ -157,6 +154,7 @@ public class Interpretation { /** * Returns the annotation with the given annotationClass (and create it if necessary). + * * @param from start of the substring * @param to end of the substring * @param annotationClass class of the annotation @@ -169,7 +167,8 @@ public class Interpretation { * Sets a key/value pair for an annotation of a substring. If an annotation of the class * does not exist, a new is created. * - * A shortcut for annotate(from, to, annotationClass, key, value + * A shortcut for annotate(from, to, annotationClass, key, value) + * * @param from start of the substring * @param to end of the substring * @param annotationClass class of the annotation @@ -184,7 +183,8 @@ public class Interpretation { * Sets a key/value pair for an annotation of a substring. If an annotation of the class * does not exist, a new is created. * - * A shortcut for annotate(from, to, annotationClass, key, value + * A shortcut for annotate(from, to, annotationClass, key, value) + * * @param from start of the substring * @param to end of the substring * @param annotationClass class of the annotation @@ -219,12 +219,7 @@ public class Interpretation { */ public List<Annotations> getAll(AnnotationClass annotationClass) { // TODO: This implementation is very inefficient because it unnecessarily collects for all classes - Map<AnnotationClass,List<Annotations>> all = getAll(); - if(all.containsKey(annotationClass)){ - return all.get(annotationClass); - } else { - return Collections.emptyList(); - } + return getAll().getOrDefault(annotationClass, List.of()); } /** @@ -256,7 +251,7 @@ public class Interpretation { /** * Gets the value of a property set on an annotation. * If the annotation or the key/value pair does not exists, null - * is returned + * is returned. */ public Object get(String annotationClass,String key) { return get(new AnnotationClass(annotationClass),key); @@ -265,7 +260,7 @@ public class Interpretation { /** * Gets the value of a property set on an annotation. * If the annotation or the key/value pair does not exists, null - * is returned + * is returned. */ public Object get(AnnotationClass annotationClass,String key) { Annotations annotations = get(annotationClass); @@ -280,7 +275,7 @@ public class Interpretation { * Equivalent to <code>get(from,to,new AnnotationClass(annotationClass))</code> */ public Annotations get(int from, int to, String annotationClass) { - return get(from,to,new AnnotationClass(annotationClass)); + return get(from, to, new AnnotationClass(annotationClass)); } /** @@ -310,7 +305,7 @@ public class Interpretation { * @return the anno */ public Annotations get(int from, int to, AnnotationClass annotationClass ) { - return rootSpan.getAnnotation(from,to,annotationClass); + return rootSpan.getAnnotation(from, to, annotationClass); } /** @@ -320,9 +315,9 @@ public class Interpretation { * is returned. * */ - public Object get(int from,int to,String annotationClass,String key) { - Annotations annotations = get(from,to,annotationClass); - if(annotations != null) { + public Object get(int from, int to, String annotationClass, String key) { + Annotations annotations = get(from, to, annotationClass); + if (annotations != null) { return annotations.get(key); } else { return null; @@ -331,7 +326,6 @@ public class Interpretation { /** * Gets all the annotationclasses that describes the text. - */ public Set<AnnotationClass> getClasses() { return rootSpan.getClasses(); @@ -399,7 +393,7 @@ public class Interpretation { } } sb.append("}"); - } } + } diff --git a/container-search/src/main/java/com/yahoo/text/interpretation/Modification.java b/container-search/src/main/java/com/yahoo/text/interpretation/Modification.java index e2fd5a5ec5c..28cf11c62b1 100644 --- a/container-search/src/main/java/com/yahoo/text/interpretation/Modification.java +++ b/container-search/src/main/java/com/yahoo/text/interpretation/Modification.java @@ -9,21 +9,14 @@ import java.util.HashMap; * This class represents a possible rewrite of an original text. Reasons for rewrite may be due to possible * spelling errors in the text or to query expansion. * - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + * @author Arne Bergene Fossaa */ public class Modification extends HashMap<String,Object>{ - /** - * - */ - private static final long serialVersionUID = -8522335044460396296L; - - public final static AnnotationClass MODIFICATION_CLASS = new AnnotationClass("modification"); - - private String text; - private Annotations annotations; + private final String text; + private final Annotations annotations; public Modification(String text) { this.text = text; diff --git a/container-search/src/main/java/com/yahoo/text/interpretation/Span.java b/container-search/src/main/java/com/yahoo/text/interpretation/Span.java index 6ade5e323e7..90cc6231d48 100644 --- a/container-search/src/main/java/com/yahoo/text/interpretation/Span.java +++ b/container-search/src/main/java/com/yahoo/text/interpretation/Span.java @@ -22,7 +22,7 @@ import java.util.Set; * <p> * A span will usually be used indirectly through Interpretation. * - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + * @author Arne Bergene Fossaa */ public class Span { @@ -33,7 +33,6 @@ public class Span { private final int from; private final int to; - /** * Creates a new root span based on the modfication */ @@ -44,7 +43,7 @@ public class Span { this.to = modification.getText().length(); } - //This constructor is private to ensure that all child spans for a span is contained inside it. + // This constructor is private to ensure that all child spans for a span is contained inside it. private Span(int from, int to, Span parent) { this.parent = parent; this.modification = parent.modification; @@ -52,8 +51,6 @@ public class Span { this.to = to; } - - /** * Returns the text that this spans is */ @@ -61,12 +58,11 @@ public class Span { return modification.getText().substring(from, to); } - + @Override public String toString() { return "SPAN: " + getText(); } - public Annotations annotate(AnnotationClass clazz) { Annotations annotations = this.annotations.get(clazz); if (!this.annotations.containsKey(clazz)) { @@ -83,7 +79,6 @@ public class Span { return addAnnotation(from, to, clazz); } - /** * Returns all annotations that are contained in either this subspan or in any of its subannotations */ @@ -124,7 +119,7 @@ public class Span { * @throws RuntimeException if (from,to) is not contained in the span */ public Annotations getAnnotation(int from, int to, AnnotationClass clazz) { - if(from < this.from || to > this.to) { + if (from < this.from || to > this.to) { throw new RuntimeException("Trying to get a range that is outside this span"); } if (this.parent != null) { @@ -140,13 +135,12 @@ public class Span { */ public Set<AnnotationClass> getClasses() { return getClasses(from, to); - } /** * Returns all AnnotationClasses that are defined for the range (from,to). * - * @throws RuntimeException if (from,to) is not contained in the span + * @throws RuntimeException if (from, to) is not contained in the span */ public Set<AnnotationClass> getClasses(int from, int to) { if(from < this.from || to > this.to) { @@ -161,8 +155,6 @@ public class Span { } } - - /** * Returns an unmodifiable list of all spans below this span that is a leaf node */ @@ -191,26 +183,25 @@ public class Span { /** hack */ public int getFrom() { return from; } + /** hack */ public int getTo() { return to; } - //Needed by addAnnotation + // Needed by addAnnotation private List<Span> getRemovableSubSpan() { return subSpans == null ? Collections.<Span>emptyList() : subSpans; } - private void addSubSpan(Span span) { - if(subSpans == null) { + if (subSpans == null) { subSpans = new ArrayList<>(); } subSpans.add(span); } - - /* + /** * How this works: * * First we check if any excisting subannotation can contain this annotation. If so, we leave it to them to add @@ -222,7 +213,7 @@ public class Span { */ private Annotations addAnnotation(int from, int to, AnnotationClass clazz) { if (equalsRange(from, to)) { - //We simply add everything from the new span to this + // We simply add everything from the new span to this if (annotations.containsKey(clazz)) { return annotations.get(clazz); } else { @@ -232,7 +223,7 @@ public class Span { } } - //We then check if any of the children intersects + // We then check if any of the children intersects for (Span subSpan : getSubSpans()) { if (subSpan.intersects(from, to)) { throw new RuntimeException("Trying to add span that intersects already excisting span"); @@ -241,14 +232,13 @@ public class Span { } } - //We now know that we have to add the new span to this span + // We now know that we have to add the new span to this span Span span = new Span(from, to, this); Annotations nAnnotations = new Annotations(span); span.annotations.put(clazz,nAnnotations); addSubSpan(span); - - //We then add any subannotation that is inside the span + // We then add any subannotation that is inside the span Iterator<Span> subIterator = getRemovableSubSpan().iterator(); while (subIterator.hasNext()) { @@ -256,7 +246,7 @@ public class Span { if (subSpan.contains(from, to)) { return subSpan.addAnnotation(from, to, clazz); } else if (subSpan.isInside(from, to)) { - //Overtake the subannotation + // Take over the subannotation subSpan.parent = span; span.addSubSpan(subSpan); subIterator.remove(); @@ -265,7 +255,6 @@ public class Span { return nAnnotations; } - private boolean contains(int from, int to) { return this.from <= from && this.to >= to; } @@ -274,12 +263,9 @@ public class Span { return this.from >= from && this.to <= to; } - private boolean intersects(int from, int to) { return (this.from < from && this.to > from && this.to < to) || (this.from < to && this.to > to && this.from > from); - - } private boolean equalsRange(int from, int to) { @@ -336,7 +322,7 @@ public class Span { if (!contains(from, to)) { return null; } - //First yourself, then the subs + // First yourself, then the subs Annotations annotations = this.annotations.get(clazz); for (Span subSpan : getSubSpans()) { Annotations subAnnotations = subSpan.getBestAnnotation(from, to, clazz); @@ -346,4 +332,5 @@ public class Span { } return annotations; } + } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java index 3c96b00d628..b9e19d6a1b6 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java @@ -15,12 +15,12 @@ import java.util.concurrent.TimeUnit; * Encapsulates all trace-related components and options used by the streaming search Searcher. * * Provides a DEFAULT static instance which has the following characteristics: - * - Approximately 1 query every 2 seconds is traced + * - Approximately 1 query every second is traced * - Trace level is set to 7 for traced queries - * - Only emits traces for queries that have timed out and where the elapsed time is at least 5x + * - Only emits traces for queries that have timed out and where the elapsed time is at least 2x * of the timeout specified in the query itself * - Emits traces to the Vespa log - * - Only 1 trace every 10 seconds may be emitted to the log + * - Only 2 traces every 10 seconds may be emitted to the log */ public class TracingOptions { @@ -50,11 +50,11 @@ public class TracingOptions { public static final TracingOptions DEFAULT; public static final int DEFAULT_TRACE_LEVEL_OVERRIDE = 7; // TODO determine appropriate trace level // Traces won't be exported unless the query has timed out with a duration that is > timeout * multiplier - public static final double TRACE_TIMEOUT_MULTIPLIER_THRESHOLD = 5.0; + public static final double TRACE_TIMEOUT_MULTIPLIER_THRESHOLD = 2.0; static { - SamplingStrategy queryTraceSampler = ProbabilisticSampleRate.withSystemDefaults(0.5); - SamplingStrategy logExportSampler = MaxSamplesPerPeriod.withSteadyClock(TimeUnit.SECONDS.toNanos(10), 1); + SamplingStrategy queryTraceSampler = ProbabilisticSampleRate.withSystemDefaults(1); + SamplingStrategy logExportSampler = MaxSamplesPerPeriod.withSteadyClock(TimeUnit.SECONDS.toNanos(10), 2); TraceExporter traceExporter = new SamplingTraceExporter(new LoggingTraceExporter(), logExportSampler); DEFAULT = new TracingOptions(queryTraceSampler, traceExporter, System::nanoTime, DEFAULT_TRACE_LEVEL_OVERRIDE, TRACE_TIMEOUT_MULTIPLIER_THRESHOLD); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 4750bac551c..24dd25c5182 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -1,11 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.streamingvisitors; +import com.yahoo.container.core.documentapi.VespaDocumentAccess; import com.yahoo.document.DocumentId; import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.select.parser.TokenMgrException; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.fs4.DocsumPacket; -import com.yahoo.log.LogLevel; import com.yahoo.messagebus.routing.Route; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; @@ -30,6 +34,7 @@ import java.math.BigInteger; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -53,15 +58,15 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { private Route route; /** The configId used to access the searchcluster. */ - private String searchClusterConfigId = null; + private String searchClusterName = null; private String documentType; /** The route to the storage cluster. */ private String storageClusterRouteSpec = null; - private String getSearchClusterConfigId() { return searchClusterConfigId; } + private String getSearchClusterName() { return searchClusterName; } private String getStorageClusterRouteSpec() { return storageClusterRouteSpec; } - public final void setSearchClusterConfigId(String clusterName) { - this.searchClusterConfigId = clusterName; + public final void setSearchClusterName(String clusterName) { + this.searchClusterName = clusterName; } public final void setDocumentType(String documentType) { this.documentType = documentType; @@ -71,16 +76,35 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { this.storageClusterRouteSpec = storageClusterRouteSpec; } - private static class VdsVisitorFactory implements VisitorFactory { + private static class VespaVisitorFactory implements VdsVisitor.VisitorSessionFactory, VisitorFactory { + + private final VespaDocumentAccess access; + + private VespaVisitorFactory(VespaDocumentAccess access) { + this.access = access; + } + + @Override + public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException { + return access.createVisitorSession(params); + } + + @Override + public LoadTypeSet getLoadTypeSet() { + return ((MessageBusDocumentAccess) access.delegate()).getParams().getLoadTypes(); + } + @Override public Visitor createVisitor(Query query, String searchCluster, Route route, String documentType, int traceLevelOverride) { - return new VdsVisitor(query, searchCluster, route, documentType, traceLevelOverride); + return new VdsVisitor(query, searchCluster, route, documentType, this, traceLevelOverride); } + } - public VdsStreamingSearcher() { - this(new VdsVisitorFactory()); + public VdsStreamingSearcher(VespaDocumentAccess access) { + this(new VespaVisitorFactory(access)); } + VdsStreamingSearcher(VisitorFactory visitorFactory) { this.visitorFactory = visitorFactory; tracingOptions = TracingOptions.DEFAULT; @@ -136,17 +160,21 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { @Override public Result doSearch2(Query query, Execution execution) { + if (query.getTimeLeft() <= 0) { + return new Result(query, ErrorMessage.createTimeout(String.format("No time left for searching (timeout=%d)", query.getTimeout()))); + } + initializeMissingQueryFields(query); if (documentSelectionQueryParameterCount(query) != 1) { return new Result(query, ErrorMessage.createBackendCommunicationError("Streaming search needs one and " + "only one of these query parameters to be set: streaming.userid, streaming.groupname, " + "streaming.selection")); } - query.trace("Routing to search cluster " + getSearchClusterConfigId() + " and document type " + documentType, 4); + query.trace("Routing to search cluster " + getSearchClusterName() + " and document type " + documentType, 4); long timeStartedNanos = tracingOptions.getClock().nanoTimeNow(); int effectiveTraceLevel = inferEffectiveQueryTraceLevel(query); - Visitor visitor = visitorFactory.createVisitor(query, getSearchClusterConfigId(), route, documentType, effectiveTraceLevel); + Visitor visitor = visitorFactory.createVisitor(query, getSearchClusterName(), route, documentType, effectiveTraceLevel); try { visitor.doSearch(); } catch (ParseException e) { @@ -163,7 +191,7 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { query.toString(), elapsedMillis / 1000.0))); } return new Result(query, ErrorMessage.createTimeout(e.getMessage())); - } catch (InterruptedException|IllegalArgumentException e) { + } catch (InterruptedException | IllegalArgumentException e) { return new Result(query, ErrorMessage.createBackendCommunicationError(e.getMessage())); } @@ -300,9 +328,9 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { String expUserId = query.properties().getString(streamingUserid); String expGroupName = query.properties().getString(streamingGroupname); - LogLevel logLevel = LogLevel.ERROR; + Level logLevel = Level.SEVERE; if (skippedEarlierResult) { - logLevel = LogLevel.DEBUG; + logLevel = Level.FINE; } DocumentId docId; diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index 795b62663d5..49fda880b44 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -17,7 +17,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Trace; import com.yahoo.messagebus.routing.Route; @@ -64,6 +64,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private static final CompoundName streamingPriority=new CompoundName("streaming.priority"); private static final CompoundName streamingMaxbucketspervisitor=new CompoundName("streaming.maxbucketspervisitor"); + protected static final int MAX_BUCKETS_PER_VISITOR = 1024; + private static final Logger log = Logger.getLogger(VdsVisitor.class.getName()); private final VisitorParameters params = new VisitorParameters(""); private List<SearchResult.Hit> hits = new ArrayList<>(); @@ -81,55 +83,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { LoadTypeSet getLoadTypeSet(); } - private static class MessageBusVisitorSessionFactory implements VisitorSessionFactory { - private static final Object initMonitor = new Object(); - private static final AtomicReference<MessageBusVisitorSessionFactory> instance = new AtomicReference<>(); - - private final LoadTypeSet loadTypes; - private final DocumentAccess access; - - private MessageBusVisitorSessionFactory() { - loadTypes = new LoadTypeSet("client"); - access = new MessageBusDocumentAccess(new MessageBusParams(loadTypes)); - } - - @Override - public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException { - return access.createVisitorSession(params); - } - - @Override - public LoadTypeSet getLoadTypeSet() { - return loadTypes; - } - - /** - * Returns a single, shared instance of this class which is lazily created in a thread-safe - * manner the first time this method is invoked. - * - * May throw any config-related exception if subscription fails. - */ - static MessageBusVisitorSessionFactory sharedInstance() { - var ref = instance.getAcquire(); - if (ref != null) { - return ref; - } - synchronized (initMonitor) { - ref = instance.getAcquire(); - if (ref != null) { - return ref; - } - ref = new MessageBusVisitorSessionFactory(); - instance.setRelease(ref); - } - return ref; - } - } - - public VdsVisitor(Query query, String searchCluster, Route route, String documentType, int traceLevelOverride) { - this(query, searchCluster, route, documentType, MessageBusVisitorSessionFactory.sharedInstance(), traceLevelOverride); - } - public VdsVisitor(Query query, String searchCluster, Route route, String documentType, VisitorSessionFactory visitorSessionFactory, int traceLevelOverride) @@ -142,9 +95,9 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private int inferSessionTraceLevel(Query query) { int implicitLevel = traceLevelOverride; - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { implicitLevel = 9; - } else if (log.isLoggable(LogLevel.DEBUG)) { + } else if (log.isLoggable(Level.FINE)) { implicitLevel = 7; } return Math.max(query.getTraceLevel(), implicitLevel); @@ -199,7 +152,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } params.setMaxPending(Integer.MAX_VALUE); - params.setMaxBucketsPerVisitor(Integer.MAX_VALUE); + params.setMaxBucketsPerVisitor(MAX_BUCKETS_PER_VISITOR); params.setTraceLevel(inferSessionTraceLevel(query)); @@ -262,9 +215,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { static int getQueryFlags(Query query) { int flags = 0; - boolean requestCoverage=true; // Always request coverage information + boolean requestCoverage = true; // Always request coverage information - flags |= 0; // was collapse flags |= query.properties().getBoolean(Model.ESTIMATE) ? 0x00000080 : 0; flags |= (query.getRanking().getFreshness() != null) ? 0x00002000 : 0; flags |= requestCoverage ? 0x00008000 : 0; @@ -329,26 +281,24 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { public void doSearch() throws InterruptedException, ParseException, TimeoutException { VisitorSession session = visitorSessionFactory.createVisitorSession(params); try { - if ( !session.waitUntilDone(query.getTimeout())) { - log.log(LogLevel.DEBUG, "Visitor returned from waitUntilDone without being completed for " + query + " with selection " + params.getDocumentSelection()); + if ( ! session.waitUntilDone(query.getTimeout())) { + log.log(Level.FINE, () -> "Visitor returned from waitUntilDone without being completed for " + query + " with selection " + params.getDocumentSelection()); session.abort(); throw new TimeoutException("Query timed out in " + VdsStreamingSearcher.class.getName()); } } finally { session.destroy(); sessionTrace = session.getTrace(); - log.log(LogLevel.DEBUG, () -> sessionTrace.toString()); + log.log(Level.FINE, () -> sessionTrace.toString()); query.trace(sessionTrace.toString(), false, 9); } if (params.getControlHandler().getResult().code == VisitorControlHandler.CompletionCode.SUCCESS) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "VdsVisitor completed successfully for " + query + " with selection " + params.getDocumentSelection()); - } + log.log(Level.FINE, () -> "VdsVisitor completed successfully for " + query + " with selection " + params.getDocumentSelection()); } else { - throw new IllegalArgumentException("Query failed: " // TODO: Is it necessary to use a runtime exception? - + params.getControlHandler().getResult().code + ": " - + params.getControlHandler().getResult().message); + throw new IllegalArgumentException("Query failed: " + + params.getControlHandler().getResult().code + ": " + + params.getControlHandler().getResult().message); } } @@ -384,8 +334,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } public void onSearchResult(SearchResult sr) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Got SearchResult for query with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Got SearchResult for query with selection " + params.getDocumentSelection()); } handleSearchResult(sr); } @@ -393,8 +343,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private void handleSearchResult(SearchResult sr) { final int hitCountTotal = sr.getTotalHitCount(); final int hitCount = sr.getHitCount(); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Got SearchResult with " + hitCountTotal + " in total and " + hitCount + " hits in real for query with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Got SearchResult with " + hitCountTotal + " in total and " + hitCount + " hits in real for query with selection " + params.getDocumentSelection()); } List<SearchResult.Hit> newHits = new ArrayList<>(hitCount); @@ -412,20 +362,21 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } private void mergeGroupingMaps(Map<Integer, byte []> newGroupingMap) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "mergeGroupingMaps: newGroupingMap = " + newGroupingMap); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "mergeGroupingMaps: newGroupingMap = " + newGroupingMap); } for(Integer key : newGroupingMap.keySet()) { byte [] value = newGroupingMap.get(key); Grouping newGrouping = new Grouping(); - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Received group with key " + key + " and size " + value.length); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Received group with key " + key + " and size " + value.length); } BufferSerializer buf = new BufferSerializer( new GrowableByteBuffer(ByteBuffer.wrap(value)) ); newGrouping.deserialize(buf); if (buf.getBuf().hasRemaining()) { - throw new IllegalArgumentException("Failed deserializing grouping. There are still data left. Position = " + buf.position() + ", limit = " + buf.getBuf().limit()); + throw new IllegalArgumentException("Failed deserializing grouping. There is still data left. " + + "Position = " + buf.position() + ", limit = " + buf.getBuf().limit()); } synchronized (groupingMap) { @@ -440,16 +391,16 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } public void onDocumentSummary(DocumentSummary ds) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Got DocumentSummary for query with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Got DocumentSummary for query with selection " + params.getDocumentSelection()); } handleSummary(ds); } private void handleSummary(DocumentSummary ds) { int summaryCount = ds.getSummaryCount(); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Got DocumentSummary with " + summaryCount + " summaries for query with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Got DocumentSummary with " + summaryCount + " summaries for query with selection " + params.getDocumentSelection()); } synchronized (summaryMap) { for (int i = 0; i < summaryCount; i++) { diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/Visitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/Visitor.java index e8b83495c69..8065f71c1f0 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/Visitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/Visitor.java @@ -15,7 +15,7 @@ import java.util.Map; /** * Visitor for performing searches and accessing results. * - * @author <a href="mailto:ulf@yahoo-inc.com">Ulf Carlin</a> + * @author Ulf Carlin */ interface Visitor { diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VisitorFactory.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VisitorFactory.java index 7ce323a2f2b..421a36dbc63 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VisitorFactory.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VisitorFactory.java @@ -7,7 +7,7 @@ import com.yahoo.search.Query; /** * A factory that creates Visitors. * - * @author <a href="mailto:ulf@yahoo-inc.com">Ulf Carlin</a> + * @author Ulf Carlin */ interface VisitorFactory { diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java index 0aaf301e071..230af8971c2 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.streamingvisitors.tracing; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.function.Supplier; import java.util.logging.Logger; @@ -17,7 +17,7 @@ public class LoggingTraceExporter implements TraceExporter { public void maybeExport(Supplier<TraceDescription> traceDescriptionSupplier) { var traceDescription = traceDescriptionSupplier.get(); if (traceDescription.getTrace() != null) { - log.log(LogLevel.WARNING, String.format("%s: %s", traceDescription.getDescription(), + log.log(Level.WARNING, String.format("%s: %s", traceDescription.getDescription(), traceDescription.getTrace().toString())); } } diff --git a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj index 3a82ffffb47..24fae895364 100644 --- a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj +++ b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj @@ -310,6 +310,7 @@ TermType termType() : <DOLLAR> { return TermType.RANK; } | <PLUS> { return TermType.AND; } | <DASH> { return TermType.NOT; } | + <EQUALS> { return TermType.EQUIV; } | { return TermType.DEFAULT; } } diff --git a/container-search/src/main/resources/configdefinitions/fs4.def b/container-search/src/main/resources/configdefinitions/container.search.fs4.def index 9562cfa75bd..9562cfa75bd 100644 --- a/container-search/src/main/resources/configdefinitions/fs4.def +++ b/container-search/src/main/resources/configdefinitions/container.search.fs4.def diff --git a/container-search/src/main/resources/configdefinitions/qr-monitor.def b/container-search/src/main/resources/configdefinitions/prelude.cluster.qr-monitor.def index 2c4ff3c6167..2c4ff3c6167 100644 --- a/container-search/src/main/resources/configdefinitions/qr-monitor.def +++ b/container-search/src/main/resources/configdefinitions/prelude.cluster.qr-monitor.def diff --git a/container-search/src/main/resources/configdefinitions/emulation.def b/container-search/src/main/resources/configdefinitions/prelude.emulation.def index 70d2d4954a4..70d2d4954a4 100644 --- a/container-search/src/main/resources/configdefinitions/emulation.def +++ b/container-search/src/main/resources/configdefinitions/prelude.emulation.def diff --git a/container-search/src/main/resources/configdefinitions/documentdb-info.def b/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def index 76096b4a6f7..76096b4a6f7 100644 --- a/container-search/src/main/resources/configdefinitions/documentdb-info.def +++ b/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def diff --git a/container-search/src/main/resources/configdefinitions/keyvalue.def b/container-search/src/main/resources/configdefinitions/prelude.searcher.keyvalue.def index 95153708aa2..95153708aa2 100644 --- a/container-search/src/main/resources/configdefinitions/keyvalue.def +++ b/container-search/src/main/resources/configdefinitions/prelude.searcher.keyvalue.def diff --git a/container-search/src/main/resources/configdefinitions/qr-quotetable.def b/container-search/src/main/resources/configdefinitions/prelude.searcher.qr-quotetable.def index 40979ad2a35..40979ad2a35 100644 --- a/container-search/src/main/resources/configdefinitions/qr-quotetable.def +++ b/container-search/src/main/resources/configdefinitions/prelude.searcher.qr-quotetable.def diff --git a/container-search/src/main/resources/configdefinitions/semantic-rules.def b/container-search/src/main/resources/configdefinitions/prelude.semantics.semantic-rules.def index 5ac0cca7ff6..5ac0cca7ff6 100644 --- a/container-search/src/main/resources/configdefinitions/semantic-rules.def +++ b/container-search/src/main/resources/configdefinitions/prelude.semantics.semantic-rules.def diff --git a/container-search/src/main/resources/configdefinitions/cluster.def b/container-search/src/main/resources/configdefinitions/search.config.cluster.def index 52eca7ef753..812073517bb 100644 --- a/container-search/src/main/resources/configdefinitions/cluster.def +++ b/container-search/src/main/resources/configdefinitions/search.config.cluster.def @@ -5,8 +5,7 @@ namespace=search.config #Note: Use clusterName where possible instead clusterId int default=0 -#Internal searcher cache. Size is measured in megabytes of raw packet -#size. Hits larger than 1% of total cache size will not be cached. +# Not used cacheSize int default=1 #Timeout for internal searcher cache. Entries older than this number diff --git a/container-search/src/main/resources/configdefinitions/index-info.def b/container-search/src/main/resources/configdefinitions/search.config.index-info.def index f3b905d4d0a..f3b905d4d0a 100644 --- a/container-search/src/main/resources/configdefinitions/index-info.def +++ b/container-search/src/main/resources/configdefinitions/search.config.index-info.def diff --git a/container-search/src/main/resources/configdefinitions/qr-start.def b/container-search/src/main/resources/configdefinitions/search.config.qr-start.def index 031877ada81..95e9d4575dd 100644 --- a/container-search/src/main/resources/configdefinitions/qr-start.def +++ b/container-search/src/main/resources/configdefinitions/search.config.qr-start.def @@ -20,6 +20,9 @@ jvm.minHeapsize int default=1536 restart ## Stack size (in kilobytes) jvm.stacksize int default=512 restart +## CompressedOOps size in megabytes +jvm.compressedClassSpaceSize int default=32 restart + ## Base value of maximum direct memory size (in megabytes) jvm.baseMaxDirectMemorySize int default=75 restart diff --git a/container-search/src/main/resources/configdefinitions/rate-limiting.def b/container-search/src/main/resources/configdefinitions/search.config.rate-limiting.def index 23235617f8c..23235617f8c 100644 --- a/container-search/src/main/resources/configdefinitions/rate-limiting.def +++ b/container-search/src/main/resources/configdefinitions/search.config.rate-limiting.def diff --git a/container-search/src/main/resources/configdefinitions/federation.def b/container-search/src/main/resources/configdefinitions/search.federation.federation.def index 36eb5d4b4c8..36eb5d4b4c8 100644 --- a/container-search/src/main/resources/configdefinitions/federation.def +++ b/container-search/src/main/resources/configdefinitions/search.federation.federation.def diff --git a/container-search/src/main/resources/configdefinitions/provider.def b/container-search/src/main/resources/configdefinitions/search.federation.provider.def index f9ab305b114..f9ab305b114 100644 --- a/container-search/src/main/resources/configdefinitions/provider.def +++ b/container-search/src/main/resources/configdefinitions/search.federation.provider.def diff --git a/container-search/src/main/resources/configdefinitions/searchchain-forward.def b/container-search/src/main/resources/configdefinitions/search.federation.searchchain-forward.def index 0e86490e120..0e86490e120 100644 --- a/container-search/src/main/resources/configdefinitions/searchchain-forward.def +++ b/container-search/src/main/resources/configdefinitions/search.federation.searchchain-forward.def diff --git a/container-search/src/main/resources/configdefinitions/strict-contracts.def b/container-search/src/main/resources/configdefinitions/search.federation.strict-contracts.def index 5ceb37db8d1..5ceb37db8d1 100644 --- a/container-search/src/main/resources/configdefinitions/strict-contracts.def +++ b/container-search/src/main/resources/configdefinitions/search.federation.strict-contracts.def diff --git a/container-search/src/main/resources/configdefinitions/search-with-renderer-handler.def b/container-search/src/main/resources/configdefinitions/search.handler.search-with-renderer-handler.def index a34e08a1c82..a34e08a1c82 100644 --- a/container-search/src/main/resources/configdefinitions/search-with-renderer-handler.def +++ b/container-search/src/main/resources/configdefinitions/search.handler.search-with-renderer-handler.def diff --git a/container-search/src/main/resources/configdefinitions/page-templates.def b/container-search/src/main/resources/configdefinitions/search.pagetemplates.page-templates.def index 31ec7644d18..31ec7644d18 100644 --- a/container-search/src/main/resources/configdefinitions/page-templates.def +++ b/container-search/src/main/resources/configdefinitions/search.pagetemplates.page-templates.def diff --git a/container-search/src/main/resources/configdefinitions/resolvers.def b/container-search/src/main/resources/configdefinitions/search.pagetemplates.resolvers.def index 6003fdf81f1..6003fdf81f1 100644 --- a/container-search/src/main/resources/configdefinitions/resolvers.def +++ b/container-search/src/main/resources/configdefinitions/search.pagetemplates.resolvers.def diff --git a/container-search/src/main/resources/configdefinitions/query-profiles.def b/container-search/src/main/resources/configdefinitions/search.query.profile.config.query-profiles.def index 3c6d11e2944..869c75df0e9 100644 --- a/container-search/src/main/resources/configdefinitions/query-profiles.def +++ b/container-search/src/main/resources/configdefinitions/search.query.profile.config.query-profiles.def @@ -52,10 +52,14 @@ queryprofile[].queryprofilevariant[].inherit[] string queryprofile[].queryprofilevariant[].property[].name string # Content of profile variant queryprofile[].queryprofilevariant[].property[].value string +# Whether this property is overridable: "true", "false" or "" +queryprofile[].queryprofilevariant[].property[].overridable string default="" # Content of profile variant queryprofile[].queryprofilevariant[].reference[].name string # Content of profile variant queryprofile[].queryprofilevariant[].reference[].value string +# Whether this reference is overridable: "true", "false" or "" +queryprofile[].queryprofilevariant[].reference[].overridable string default="" # A query profile type defines the values of instance query profiles. # The id follows the same rules as for query profiles diff --git a/container-search/src/main/resources/configdefinitions/rewrites.def b/container-search/src/main/resources/configdefinitions/search.query.rewrite.rewrites.def index ecca422342a..ecca422342a 100644 --- a/container-search/src/main/resources/configdefinitions/rewrites.def +++ b/container-search/src/main/resources/configdefinitions/search.query.rewrite.rewrites.def diff --git a/container-search/src/main/resources/configdefinitions/lowercasing.def b/container-search/src/main/resources/configdefinitions/search.querytransform.lowercasing.def index b656c451e11..b656c451e11 100644 --- a/container-search/src/main/resources/configdefinitions/lowercasing.def +++ b/container-search/src/main/resources/configdefinitions/search.querytransform.lowercasing.def diff --git a/container-search/src/main/resources/configdefinitions/measure-qps.def b/container-search/src/main/resources/configdefinitions/search.statistics.measure-qps.def index c8b38b9db6e..c8b38b9db6e 100644 --- a/container-search/src/main/resources/configdefinitions/measure-qps.def +++ b/container-search/src/main/resources/configdefinitions/search.statistics.measure-qps.def diff --git a/container-search/src/main/resources/configdefinitions/timing-searcher.def b/container-search/src/main/resources/configdefinitions/search.statistics.timing-searcher.def index 7c2b698bdb0..7c2b698bdb0 100644 --- a/container-search/src/main/resources/configdefinitions/timing-searcher.def +++ b/container-search/src/main/resources/configdefinitions/search.statistics.timing-searcher.def |