From 6f8a35e84e54ca0b1a78f02f18bf8018479c0a58 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 29 Mar 2021 09:09:23 +0200 Subject: Remove unsupported constructs --- .../main/antlr4/com/yahoo/search/yql/yqlplus.g4 | 136 +------------ .../java/com/yahoo/search/yql/ProgramParser.java | 213 +-------------------- 2 files changed, 11 insertions(+), 338 deletions(-) (limited to 'container-search') diff --git a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 index b7d443ea56c..26357267f4e 100644 --- a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 +++ b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 @@ -18,22 +18,9 @@ options { protected Stack expression_stack = new Stack(); } -// tokens for command syntax - CREATE : 'create'; +// tokens + SELECT : 'select'; - INSERT : 'insert'; - UPDATE : 'update'; - SET : 'set'; - VIEW : 'view'; - TABLE : 'table'; - DELETE : 'delete'; - INTO : 'into'; - VALUES : 'values'; - IMPORT : 'import'; - NEXT : 'next'; - PAGED : 'paged'; - FALLBACK : 'fallback'; - IMPORT_FROM :; LIMIT : 'limit'; OFFSET : 'offset'; @@ -44,36 +31,12 @@ options { FROM : 'from'; SOURCES : 'sources'; AS : 'as'; - MERGE : 'merge'; - LEFT : 'left'; - JOIN : 'join'; - - ON : 'on'; + + ON : 'on'; // TODO: not used? COMMA : ','; OUTPUT : 'output'; COUNT : 'count'; - RETURNING : 'returning'; - APPLY : 'apply'; - CAST : 'cast'; - - BEGIN : 'begin'; - END : 'end'; - - // type-related - TYPE_BYTE : 'byte'; - TYPE_INT16 : 'int16'; - TYPE_INT32 : 'int32'; - TYPE_INT64 : 'int64'; - TYPE_STRING : 'string'; - TYPE_DOUBLE : 'double'; - TYPE_TIMESTAMP : 'timestamp'; - TYPE_BOOLEAN : 'boolean'; - TYPE_ARRAY : 'array'; - TYPE_MAP : 'map'; - - // READ_FIELD; - - // token literals + TRUE : 'true'; FALSE : 'false'; @@ -123,7 +86,6 @@ options { // statement delimiter SEMI : ';'; - PROGRAM : 'program'; TIMEOUT : 'timeout'; @@ -219,28 +181,12 @@ ident ; keyword_as_ident - : SELECT | TABLE | DELETE | INTO | VALUES | LIMIT | OFFSET | WHERE | 'order' | 'by' | DESC | MERGE | LEFT | JOIN - | ON | OUTPUT | COUNT | BEGIN | END | APPLY | TYPE_BYTE | TYPE_INT16 | TYPE_INT32 | TYPE_INT64 | TYPE_BOOLEAN | TYPE_TIMESTAMP | TYPE_DOUBLE | TYPE_STRING | TYPE_ARRAY | TYPE_MAP - | VIEW | CREATE | IMPORT | PROGRAM | NEXT | PAGED | SOURCES | SET | MATCHES | LIKE | CAST + : SELECT | LIMIT | OFFSET | WHERE | 'order' | 'by' | DESC | ON | OUTPUT | COUNT | SOURCES | MATCHES | LIKE ; -program : params? (import_statement SEMI)* (ddl SEMI)* (statement SEMI)* EOF +program : (statement SEMI)* EOF ; -params - : PROGRAM LPAREN program_arglist? RPAREN SEMI - ; - -import_statement - : IMPORT moduleName AS moduleId - | IMPORT moduleId - | FROM moduleName IMPORT import_list - ; - -import_list - : moduleId (',' moduleId)* - ; - moduleId : ID ; @@ -250,46 +196,18 @@ moduleName | namespaced_name ; -ddl - : view - ; - -view : CREATE VIEW ID AS source_statement - ; - -program_arglist - : procedure_argument (',' procedure_argument)* - ; - -procedure_argument - : - AT (ident TYPE_ARRAY LT typename GTEQ (expression[false])? ) {registerParameter($ident.start.getText(), $typename.start.getText());} - | AT (ident typename ('=' expression[false])? ) {registerParameter($ident.start.getText(), $typename.start.getText());} - ; - statement : output_statement - | selectvar_statement - | next_statement ; output_statement - : source_statement paged_clause? output_spec? - ; - -paged_clause - : PAGED fixed_or_parameter + : source_statement output_spec? ; -next_statement - : NEXT literalString OUTPUT AS ident - ; - source_statement : query_statement (PIPE pipeline_step)* ; - pipeline_step : namespaced_name arguments[false]? | vespa_grouping @@ -300,50 +218,20 @@ vespa_grouping | annotation VESPA_GROUPING ; -selectvar_statement - : CREATE ('temp' | 'temporary') TABLE ident AS LPAREN source_statement RPAREN - ; - -typename - : TYPE_BYTE | TYPE_INT16 | TYPE_INT32 | TYPE_INT64 | TYPE_STRING | TYPE_BOOLEAN | TYPE_TIMESTAMP - | arrayType | mapType | TYPE_DOUBLE - ; - -arrayType - : TYPE_ARRAY LT typename GT - ; - -mapType - : TYPE_MAP LT typename GT - ; - output_spec : (OUTPUT AS ident) | (OUTPUT COUNT AS ident) ; query_statement - : merge_statement - | select_statement + : select_statement | insert_statement | delete_statement | update_statement ; -// This does not use the UNION / UNION ALL from SQL because the semantics are different than SQL UNION -// - no set operation is implied (no DISTINCT) -// - CQL resultsets may be heterogeneous (rows may have heterogenous types) -merge_statement - : merge_component (MERGE merge_component)+ - ; - -merge_component - : select_statement - | LPAREN source_statement RPAREN - ; - select_statement - : SELECT select_field_spec select_source? where? orderby? limit? offset? timeout? fallback? + : SELECT select_field_spec select_source? where? orderby? limit? offset? timeout? ; select_field_spec @@ -355,10 +243,6 @@ project_spec : field_def (COMMA field_def)* ; -fallback - : FALLBACK select_statement - ; - timeout : TIMEOUT fixed_or_parameter ; diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index bcd5822a6f6..7fa8d05bb04 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -14,7 +14,6 @@ import com.yahoo.search.yql.yqlplusParser.AnnotateExpressionContext; import com.yahoo.search.yql.yqlplusParser.ArgumentContext; import com.yahoo.search.yql.yqlplusParser.ArgumentsContext; import com.yahoo.search.yql.yqlplusParser.ArrayLiteralContext; -import com.yahoo.search.yql.yqlplusParser.ArrayTypeContext; import com.yahoo.search.yql.yqlplusParser.Call_sourceContext; import com.yahoo.search.yql.yqlplusParser.ConstantArrayContext; import com.yahoo.search.yql.yqlplusParser.ConstantExpressionContext; @@ -24,14 +23,11 @@ import com.yahoo.search.yql.yqlplusParser.Delete_statementContext; import com.yahoo.search.yql.yqlplusParser.DereferencedExpressionContext; import com.yahoo.search.yql.yqlplusParser.EqualityExpressionContext; import com.yahoo.search.yql.yqlplusParser.ExpressionContext; -import com.yahoo.search.yql.yqlplusParser.FallbackContext; import com.yahoo.search.yql.yqlplusParser.Field_defContext; import com.yahoo.search.yql.yqlplusParser.Field_names_specContext; import com.yahoo.search.yql.yqlplusParser.Field_values_group_specContext; import com.yahoo.search.yql.yqlplusParser.Field_values_specContext; import com.yahoo.search.yql.yqlplusParser.IdentContext; -import com.yahoo.search.yql.yqlplusParser.Import_listContext; -import com.yahoo.search.yql.yqlplusParser.Import_statementContext; import com.yahoo.search.yql.yqlplusParser.InNotInTargetContext; import com.yahoo.search.yql.yqlplusParser.Insert_sourceContext; import com.yahoo.search.yql.yqlplusParser.Insert_statementContext; @@ -44,23 +40,13 @@ import com.yahoo.search.yql.yqlplusParser.Literal_listContext; import com.yahoo.search.yql.yqlplusParser.LogicalANDExpressionContext; import com.yahoo.search.yql.yqlplusParser.LogicalORExpressionContext; import com.yahoo.search.yql.yqlplusParser.MapExpressionContext; -import com.yahoo.search.yql.yqlplusParser.MapTypeContext; -import com.yahoo.search.yql.yqlplusParser.Merge_componentContext; -import com.yahoo.search.yql.yqlplusParser.Merge_statementContext; -import com.yahoo.search.yql.yqlplusParser.ModuleIdContext; -import com.yahoo.search.yql.yqlplusParser.ModuleNameContext; import com.yahoo.search.yql.yqlplusParser.MultiplicativeExpressionContext; import com.yahoo.search.yql.yqlplusParser.Namespaced_nameContext; -import com.yahoo.search.yql.yqlplusParser.Next_statementContext; import com.yahoo.search.yql.yqlplusParser.OffsetContext; import com.yahoo.search.yql.yqlplusParser.OrderbyContext; import com.yahoo.search.yql.yqlplusParser.Orderby_fieldContext; import com.yahoo.search.yql.yqlplusParser.Output_specContext; -import com.yahoo.search.yql.yqlplusParser.Paged_clauseContext; -import com.yahoo.search.yql.yqlplusParser.ParamsContext; import com.yahoo.search.yql.yqlplusParser.Pipeline_stepContext; -import com.yahoo.search.yql.yqlplusParser.Procedure_argumentContext; -import com.yahoo.search.yql.yqlplusParser.Program_arglistContext; import com.yahoo.search.yql.yqlplusParser.Project_specContext; import com.yahoo.search.yql.yqlplusParser.ProgramContext; import com.yahoo.search.yql.yqlplusParser.PropertyNameAndValueContext; @@ -72,18 +58,15 @@ import com.yahoo.search.yql.yqlplusParser.Scalar_literalContext; import com.yahoo.search.yql.yqlplusParser.Select_source_joinContext; import com.yahoo.search.yql.yqlplusParser.Select_source_multiContext; import com.yahoo.search.yql.yqlplusParser.Select_statementContext; -import com.yahoo.search.yql.yqlplusParser.Selectvar_statementContext; import com.yahoo.search.yql.yqlplusParser.Sequence_sourceContext; import com.yahoo.search.yql.yqlplusParser.Source_listContext; import com.yahoo.search.yql.yqlplusParser.Source_specContext; import com.yahoo.search.yql.yqlplusParser.Source_statementContext; import com.yahoo.search.yql.yqlplusParser.StatementContext; import com.yahoo.search.yql.yqlplusParser.TimeoutContext; -import com.yahoo.search.yql.yqlplusParser.TypenameContext; import com.yahoo.search.yql.yqlplusParser.UnaryExpressionContext; import com.yahoo.search.yql.yqlplusParser.Update_statementContext; import com.yahoo.search.yql.yqlplusParser.Update_valuesContext; -import com.yahoo.search.yql.yqlplusParser.ViewContext; import com.yahoo.search.yql.yqlplusParser.WhereContext; import org.antlr.v4.runtime.BaseErrorListener; @@ -175,34 +158,11 @@ final class ProgramParser { } } - public OperatorNode parse(String programName, InputStream program) throws IOException, RecognitionException { - yqlplusParser parser = prepareParser(programName, program); - return convertProgram(parseProgram(parser), parser, programName); - } - public OperatorNode parse(String programName, String program) throws IOException, RecognitionException { yqlplusParser parser = prepareParser(programName, program); return convertProgram(parseProgram(parser), parser, programName); } - public OperatorNode parse(File input) throws IOException, RecognitionException { - yqlplusParser parser = prepareParser(input); - return convertProgram(parseProgram(parser), parser, input.getAbsoluteFile().toString()); - } - - public OperatorNode parseExpression(String input) throws IOException, RecognitionException { - return convertExpr(prepareParser("", input).expression(false).getRuleContext(), new Scope()); - } - - public OperatorNode parseExpression(String input, Set visibleAliases) throws IOException, RecognitionException { - Scope scope = new Scope(); - final Location loc = new Location("", -1, -1); - for (String alias : visibleAliases) { - scope.defineDataSource(loc, alias); - } - return convertExpr(prepareParser("", input).expression(false).getRuleContext(), scope); - } - private Location toLocation(Scope scope, ParseTree node) { Token start; if (node instanceof ParserRuleContext) { @@ -381,7 +341,7 @@ final class ProgramParser { Preconditions.checkArgument(node instanceof Select_statementContext || node instanceof Insert_statementContext || node instanceof Update_statementContext || node instanceof Delete_statementContext); - // SELECT^ select_field_spec select_source where? orderby? limit? offset? timeout? fallback? + // SELECT^ select_field_spec select_source where? orderby? limit? offset? timeout? // select is the only place to define where/orderby/limit/offset and joins Scope scope = scopeParent.child(); ProjectionBuilder proj = null; @@ -391,7 +351,6 @@ final class ProgramParser { OperatorNode offset = null; OperatorNode limit = null; OperatorNode timeout = null; - OperatorNode fallback = null; OperatorNode insertValues = null; OperatorNode updateValues = null; @@ -475,9 +434,6 @@ final class ProgramParser { case yqlplusParser.RULE_timeout: timeout = convertExpr(((TimeoutContext) child).fixed_or_parameter(), scope); break; - case yqlplusParser.RULE_fallback: - fallback = convertQuery(((FallbackContext) child).select_statement(), scope); - break; case yqlplusParser.RULE_insert_values: if (child.getChild(0) instanceof yqlplusParser.Query_statementContext) { insertValues = convertQuery(child.getChild(0).getChild(0), scope); @@ -558,10 +514,6 @@ final class ProgramParser { if (timeout != null) { result = OperatorNode.create(SequenceOperator.TIMEOUT, result, timeout); } - // if there's a fallback, emit a fallback node - if (fallback != null) { - result = OperatorNode.create(SequenceOperator.FALLBACK, result, fallback); - } return result; } @@ -621,21 +573,6 @@ final class ProgramParser { return result; } - private OperatorNode convertMerge(List mergeComponentList, Scope scope) { - Preconditions.checkArgument(mergeComponentList != null); - List> sources = Lists.newArrayListWithExpectedSize(mergeComponentList.size()); - for (Merge_componentContext mergeComponent:mergeComponentList) { - Select_statementContext selectContext = mergeComponent.select_statement(); - Source_statementContext sourceContext = mergeComponent.source_statement(); - if (selectContext != null) { - sources.add(convertQuery(selectContext, scope.getRoot())); - } else { - sources.add(convertQuery(sourceContext, scope.getRoot())); - } - } - return OperatorNode.create(SequenceOperator.MERGE, sources); - } - private OperatorNode convertQuery(ParseTree node, Scope scope) { if (node instanceof Select_statementContext || node instanceof Insert_statementContext @@ -645,8 +582,6 @@ final class ProgramParser { } else if (node instanceof Source_statementContext) { // for pipe Source_statementContext sourceStatementContext = (Source_statementContext)node; return convertPipe(sourceStatementContext.query_statement(), sourceStatementContext.pipeline_step(), scope); - } else if (node instanceof Merge_statementContext) { - return convertMerge(((Merge_statementContext)node).merge_component(), scope); } else { throw new IllegalArgumentException("Unexpected argument type to convertQueryStatement: " + node.toStringTree()); } @@ -763,58 +698,6 @@ final class ProgramParser { return result; } - private OperatorNode decodeType(Scope scope, TypenameContext type) { - - TypeOperator op; - ParseTree typeNode = type.getChild(0); - switch (getParseTreeIndex(typeNode)) { - case yqlplusParser.TYPE_BOOLEAN: - op = TypeOperator.BOOLEAN; - break; - case yqlplusParser.TYPE_BYTE: - op = TypeOperator.BYTE; - break; - case yqlplusParser.TYPE_DOUBLE: - op = TypeOperator.DOUBLE; - break; - case yqlplusParser.TYPE_INT16: - op = TypeOperator.INT16; - break; - case yqlplusParser.TYPE_INT32: - op = TypeOperator.INT32; - break; - case yqlplusParser.TYPE_INT64: - op = TypeOperator.INT64; - break; - case yqlplusParser.TYPE_STRING: - op = TypeOperator.STRING; - break; - case yqlplusParser.TYPE_TIMESTAMP: - op = TypeOperator.TIMESTAMP; - break; - case yqlplusParser.RULE_arrayType: - return OperatorNode.create(toLocation(scope, typeNode), TypeOperator.ARRAY, decodeType(scope, ((ArrayTypeContext)typeNode).getChild(TypenameContext.class, 0))); - case yqlplusParser.RULE_mapType: - return OperatorNode.create(toLocation(scope, typeNode), TypeOperator.MAP, decodeType(scope, ((MapTypeContext)typeNode).getChild(TypenameContext.class, 0))); - default: - throw new ProgramCompileException("Unknown type " + typeNode.getText()); - } - return OperatorNode.create(toLocation(scope, typeNode), op); - } - - private List createBindingName(ParseTree node) { - if (node instanceof ModuleNameContext) { - if (((ModuleNameContext)node).namespaced_name() != null) { - return readName(((ModuleNameContext)node).namespaced_name()); - } else if (((ModuleNameContext)node).literalString() != null) { - return ImmutableList.of(((ModuleNameContext)node).literalString().STRING().getText()); - } - } else if (node instanceof ModuleIdContext) { - return ImmutableList.of(node.getText()); - } - throw new ProgramCompileException("Wrong context"); - } - private OperatorNode convertProgram( ParserRuleContext program, yqlplusParser parser, String programName) { Scope scope = new Scope(parser, programName); @@ -826,99 +709,12 @@ final class ProgramParser { } ParserRuleContext ruleContext = (ParserRuleContext) node; switch (ruleContext.getRuleIndex()) { - case yqlplusParser.RULE_params: { - // ^(ARGUMENT ident typeref expression?) - ParamsContext paramsContext = (ParamsContext) ruleContext; - Program_arglistContext program_arglistContext = paramsContext.program_arglist(); - if (program_arglistContext != null) { - List argList = program_arglistContext.procedure_argument(); - for (Procedure_argumentContext procedureArgumentContext : argList) { - String name = procedureArgumentContext.ident().getText(); - OperatorNode type = decodeType(scope, procedureArgumentContext.getChild(TypenameContext.class, 0)); - OperatorNode defaultValue = OperatorNode.create(ExpressionOperator.NULL); - if (procedureArgumentContext.expression() != null) { - defaultValue = convertExpr(procedureArgumentContext.expression(), scope); - } - scope.defineVariable(toLocation(scope, procedureArgumentContext), name); - stmts.add(OperatorNode.create(StatementOperator.ARGUMENT, name, type, defaultValue)); - } - } - break; - } - case yqlplusParser.RULE_import_statement: { - Import_statementContext importContext = (Import_statementContext) ruleContext; - if (null == importContext.import_list()) { - List name = createBindingName(node.getChild(1)); - String target; - Location location = toLocation(scope, node.getChild(1)); - if (node.getChildCount() == 2) { - target = name.get(0); - } else if (node.getChildCount() == 4) { - target = node.getChild(3).getText(); - } else { - throw new ProgramCompileException("Unknown node count for IMPORT: " + node.toStringTree()); - } - scope.bindModule(location, name, target); - } else { - // | FROM moduleName IMPORT import_list -> ^(IMPORT_FROM - // moduleName import_list+) - Import_listContext importListContext = importContext.import_list(); - List name = createBindingName(importContext.moduleName()); - Location location = toLocation(scope, importContext.moduleName()); - List moduleIds = importListContext.moduleId(); - List symbols = Lists.newArrayListWithExpectedSize(moduleIds.size()); - for (ModuleIdContext cnode : moduleIds) { - symbols.add(cnode.ID().getText()); - } - for (String sym : symbols) { - scope.bindModuleSymbol(location, name, sym, sym); - } - } - break; - } - // DDL - case yqlplusParser.RULE_ddl: - ruleContext = (ParserRuleContext)ruleContext.getChild(0); - break; - case yqlplusParser.RULE_view: { - // view and projection expansion now has to be done by the - // execution engine - // since views/projections, in order to be useful, have to - // support being used from outside the same program - ViewContext viewContext = (ViewContext) ruleContext; - Location loc = toLocation(scope, viewContext); - scope.getRoot().defineView(loc, viewContext.ID().getText()); - stmts.add(OperatorNode.create(loc, StatementOperator.DEFINE_VIEW, viewContext.ID().getText(), convertQuery(viewContext.source_statement(), scope.getRoot()))); - break; - } case yqlplusParser.RULE_statement: { // ^(STATEMENT_QUERY source_statement paged_clause? // output_spec?) StatementContext statementContext = (StatementContext) ruleContext; switch (getParseTreeIndex(ruleContext.getChild(0))) { - case yqlplusParser.RULE_selectvar_statement: { - // ^(STATEMENT_SELECTVAR ident source_statement) - Selectvar_statementContext selectVarContext = (Selectvar_statementContext) ruleContext.getChild(0); - String variable = selectVarContext.ident().getText(); - OperatorNode query = convertQuery(selectVarContext.source_statement(), scope); - Location location = toLocation(scope, selectVarContext.ident()); - scope.defineVariable(location, variable); - stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable)); - break; - } - case yqlplusParser.RULE_next_statement: { - // NEXT^ literalString OUTPUT! AS! ident - Next_statementContext nextStateContext = (Next_statementContext) ruleContext.getChild(0); - String continuationValue = StringUnescaper.unquote(nextStateContext.literalString().getText()); - String variable = nextStateContext.ident().getText(); - Location location = toLocation(scope, node); - OperatorNode next = OperatorNode.create(location, SequenceOperator.NEXT, continuationValue); - stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, next, variable)); - stmts.add(OperatorNode.create(location, StatementOperator.OUTPUT, variable)); - scope.defineVariable(location, variable); - break; - } case yqlplusParser.RULE_output_statement: Source_statementContext source_statement = statementContext.output_statement().source_statement(); OperatorNode query; @@ -935,10 +731,6 @@ final class ProgramParser { for (int i = 1; i < outputStatement.getChildCount(); ++i) { ParseTree child = outputStatement.getChild(i); switch (getParseTreeIndex(child)) { - case yqlplusParser.RULE_paged_clause: - Paged_clauseContext pagedContext = (Paged_clauseContext) child; - pageSize = convertExpr(pagedContext.fixed_or_parameter(), scope); - break; case yqlplusParser.RULE_output_spec: Output_specContext outputSpecContext = (Output_specContext) child; variable = outputSpecContext.ident().getText(); @@ -951,9 +743,6 @@ final class ProgramParser { } } scope.defineVariable(location, variable); - if (pageSize != null) { - query = OperatorNode.create(SequenceOperator.PAGE, query, pageSize); - } stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable)); stmts.add(OperatorNode.create(location, isCountVariable ? StatementOperator.COUNT:StatementOperator.OUTPUT, variable)); } -- cgit v1.2.3