diff options
Diffstat (limited to 'container-search')
23 files changed, 690 insertions, 1203 deletions
diff --git a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 index b7d443ea56c..c0cf293f2ea 100644 --- a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 +++ b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 @@ -18,22 +18,9 @@ options { protected Stack<expression_scope> expression_stack = new Stack(); } -// tokens for command syntax - CREATE : 'create'; +// tokens + SELECT : 'select'; - INSERT : 'insert'; - UPDATE : 'update'; - SET : 'set'; - VIEW : 'view'; - TABLE : 'table'; - DELETE : 'delete'; - INTO : 'into'; - VALUES : 'values'; - IMPORT : 'import'; - NEXT : 'next'; - PAGED : 'paged'; - FALLBACK : 'fallback'; - IMPORT_FROM :; LIMIT : 'limit'; OFFSET : 'offset'; @@ -44,36 +31,11 @@ options { FROM : 'from'; SOURCES : 'sources'; AS : 'as'; - MERGE : 'merge'; - LEFT : 'left'; - JOIN : 'join'; - - ON : 'on'; + COMMA : ','; OUTPUT : 'output'; COUNT : 'count'; - RETURNING : 'returning'; - APPLY : 'apply'; - CAST : 'cast'; - - BEGIN : 'begin'; - END : 'end'; - - // type-related - TYPE_BYTE : 'byte'; - TYPE_INT16 : 'int16'; - TYPE_INT32 : 'int32'; - TYPE_INT64 : 'int64'; - TYPE_STRING : 'string'; - TYPE_DOUBLE : 'double'; - TYPE_TIMESTAMP : 'timestamp'; - TYPE_BOOLEAN : 'boolean'; - TYPE_ARRAY : 'array'; - TYPE_MAP : 'map'; - - // READ_FIELD; - - // token literals + TRUE : 'true'; FALSE : 'false'; @@ -123,7 +85,6 @@ options { // statement delimiter SEMI : ';'; - PROGRAM : 'program'; TIMEOUT : 'timeout'; @@ -149,7 +110,6 @@ FLOAT fragment EXPONENT : ('e'|'E') ('+'|'-')? ('0'..'9')+ ; - fragment DIGIT : '0'..'9' ; @@ -163,7 +123,6 @@ STRING : '"' ( ESC_SEQ | ~('\\'| '"') )* '"' | '\'' ( ESC_SEQ | ~('\\' | '\'') )* '\'' ; -///////////////////////////// fragment HEX_DIGIT : ('0'..'9'|'a'..'f'|'A'..'F') ; @@ -183,13 +142,11 @@ WS : ( ' ' | '\t' | '\r' | '\n' - // ) {$channel=HIDDEN;} ) -> channel(HIDDEN) ; COMMENT : ( ('//') ~('\n'|'\r')* '\r'? '\n'? - // | '/*' ( options {greedy=false;} : . )* '*/' | '/*' .*? '*/' ) -> channel(HIDDEN) @@ -208,39 +165,20 @@ VESPA_GROUPING_ARG (')' | ']' | '>') ; -/*------------------------------------------------------------------ - * PARSER RULES - *------------------------------------------------------------------*/ +// --------- parser rules ------------ ident : keyword_as_ident //{addChild(new TerminalNodeImpl(keyword_as_ident.getText()));} - //{return ID<IDNode>[$keyword_as_ident.text];} | ID ; keyword_as_ident - : SELECT | TABLE | DELETE | INTO | VALUES | LIMIT | OFFSET | WHERE | 'order' | 'by' | DESC | MERGE | LEFT | JOIN - | ON | OUTPUT | COUNT | BEGIN | END | APPLY | TYPE_BYTE | TYPE_INT16 | TYPE_INT32 | TYPE_INT64 | TYPE_BOOLEAN | TYPE_TIMESTAMP | TYPE_DOUBLE | TYPE_STRING | TYPE_ARRAY | TYPE_MAP - | VIEW | CREATE | IMPORT | PROGRAM | NEXT | PAGED | SOURCES | SET | MATCHES | LIKE | CAST + : SELECT | LIMIT | OFFSET | WHERE | 'order' | 'by' | DESC | OUTPUT | COUNT | SOURCES | MATCHES | LIKE ; -program : params? (import_statement SEMI)* (ddl SEMI)* (statement SEMI)* EOF +program : (statement SEMI)* EOF ; -params - : PROGRAM LPAREN program_arglist? RPAREN SEMI - ; - -import_statement - : IMPORT moduleName AS moduleId - | IMPORT moduleId - | FROM moduleName IMPORT import_list - ; - -import_list - : moduleId (',' moduleId)* - ; - moduleId : ID ; @@ -250,46 +188,18 @@ moduleName | namespaced_name ; -ddl - : view - ; - -view : CREATE VIEW ID AS source_statement - ; - -program_arglist - : procedure_argument (',' procedure_argument)* - ; - -procedure_argument - : - AT (ident TYPE_ARRAY LT typename GTEQ (expression[false])? ) {registerParameter($ident.start.getText(), $typename.start.getText());} - | AT (ident typename ('=' expression[false])? ) {registerParameter($ident.start.getText(), $typename.start.getText());} - ; - statement : output_statement - | selectvar_statement - | next_statement ; output_statement - : source_statement paged_clause? output_spec? - ; - -paged_clause - : PAGED fixed_or_parameter + : source_statement output_spec? ; -next_statement - : NEXT literalString OUTPUT AS ident - ; - source_statement : query_statement (PIPE pipeline_step)* ; - pipeline_step : namespaced_name arguments[false]? | vespa_grouping @@ -300,50 +210,17 @@ vespa_grouping | annotation VESPA_GROUPING ; -selectvar_statement - : CREATE ('temp' | 'temporary') TABLE ident AS LPAREN source_statement RPAREN - ; - -typename - : TYPE_BYTE | TYPE_INT16 | TYPE_INT32 | TYPE_INT64 | TYPE_STRING | TYPE_BOOLEAN | TYPE_TIMESTAMP - | arrayType | mapType | TYPE_DOUBLE - ; - -arrayType - : TYPE_ARRAY LT typename GT - ; - -mapType - : TYPE_MAP LT typename GT - ; - output_spec : (OUTPUT AS ident) | (OUTPUT COUNT AS ident) ; query_statement - : merge_statement - | select_statement - | insert_statement - | delete_statement - | update_statement + : select_statement ; -// This does not use the UNION / UNION ALL from SQL because the semantics are different than SQL UNION -// - no set operation is implied (no DISTINCT) -// - CQL resultsets may be heterogeneous (rows may have heterogenous types) -merge_statement - : merge_component (MERGE merge_component)+ - ; - -merge_component - : select_statement - | LPAREN source_statement RPAREN - ; - select_statement - : SELECT select_field_spec select_source? where? orderby? limit? offset? timeout? fallback? + : SELECT select_field_spec select_source? where? orderby? limit? offset? timeout? ; select_field_spec @@ -355,10 +232,6 @@ project_spec : field_def (COMMA field_def)* ; -fallback - : FALLBACK select_statement - ; - timeout : TIMEOUT fixed_or_parameter ; @@ -366,7 +239,7 @@ timeout select_source : select_source_all | select_source_multi - | select_source_join + | select_source_from ; select_source_all @@ -377,23 +250,14 @@ select_source_multi : FROM SOURCES source_list ; -select_source_join - : FROM source_spec join_expr* +select_source_from + : FROM source_spec ; source_list : namespaced_name (COMMA namespaced_name )* ; -join_expr - : (join_spec source_spec ON joinExpression) - ; - -join_spec - : LEFT JOIN - | 'inner'? JOIN - ; - source_spec : ( data_source (alias_def { ($data_source.ctx).addChild($alias_def.ctx); })? ) ; @@ -466,21 +330,6 @@ argument[boolean in_select] : expression[$in_select] ; -// -------- join expressions ------------ - -// Limit expression syntax for joins: A single equality test and one field from each source. -// This means it can always turn the join into a query to one source, collecting all of the -// keys from the results, and then a query to the other source (or querying the other source inline). -// Does not support map or index references. - -joinExpression - : joinDereferencedExpression EQ joinDereferencedExpression - ; - -joinDereferencedExpression - : namespaced_name - ; - // --------- expressions ------------ expression [boolean select] @@ -587,15 +436,6 @@ indexref[boolean in_select] propertyref : DOT nm=ID ; -operatorCall -@init{ - boolean in_select = expression_stack.peek().in_select; -} - : multOp arguments[in_select] - | additiveOp arguments[in_select] - | AND arguments[in_select] - | OR arguments[in_select] - ; primaryExpression @init { @@ -671,7 +511,7 @@ array_parameter ; literal_list - : LPAREN literal_element (COMMA literal_element)* RPAREN //{return ^(ARRAY_LITERAL literal_element+);} + : LPAREN literal_element (COMMA literal_element)* RPAREN ; literal_element @@ -683,63 +523,3 @@ fixed_or_parameter : INT | parameter ; - -// INSERT - -insert_statement - : INSERT insert_source insert_values returning_spec? - ; - -insert_source - : INTO write_data_source - ; - -write_data_source - : namespaced_name - ; - -insert_values - : field_names_spec VALUES field_values_group_spec (COMMA field_values_group_spec)* - | query_statement - ; - -field_names_spec - : LPAREN field_def (COMMA field_def)* RPAREN - ; - -field_values_spec - : LPAREN expression[true] (COMMA expression[true])* RPAREN - ; - -field_values_group_spec - : LPAREN expression[true] (COMMA expression[true])* RPAREN - ; - -returning_spec - : RETURNING select_field_spec - ; - -// DELETE - -delete_statement - : DELETE delete_source where? returning_spec? - ; - -delete_source - : FROM write_data_source - ; - -// UPDATE - -update_statement - : UPDATE update_source SET update_values where? returning_spec? - ; - -update_source - : write_data_source - ; - -update_values - : field_names_spec EQ field_values_spec - | field_def (COMMA field_def)* - ; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java index 88bae76b26d..2bf20bf7c5a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java @@ -94,7 +94,7 @@ public class QueryCanonicalizer { if (composite instanceof RankItem || composite instanceof NotItem) { collapseLevels(composite, composite.getItemIterator()); // collapse the first item only } - else if (composite instanceof AndItem || composite instanceof OrItem) { + else if (composite instanceof AndItem || composite instanceof OrItem || composite instanceof WeakAndItem) { for (ListIterator<Item> i = composite.getItemIterator(); i.hasNext(); ) collapseLevels(composite, i); } @@ -106,10 +106,17 @@ public class QueryCanonicalizer { Item child = i.next(); if (child == null) return; if (child.getClass() != composite.getClass()) return; + if (child instanceof WeakAndItem && !equalWeakAndSettings((WeakAndItem)child, (WeakAndItem)composite)) return; i.remove(); moveChildren((CompositeItem) child, i); } - + + private static boolean equalWeakAndSettings(WeakAndItem a, WeakAndItem b) { + if ( ! a.getIndexName().equals(b.getIndexName())) return false; + if (a.getN() != b.getN()) return false; + return true; + } + private static void moveChildren(CompositeItem from, ListIterator<Item> toIterator) { for (ListIterator<Item> i = from.getItemIterator(); i.hasNext(); ) toIterator.add(i.next()); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java index 8cce8fb5720..c5679e113f1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java @@ -99,13 +99,10 @@ public class WandItem extends WeightedSetItem { protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); - buffer.append(targetNumHits); - buffer.append(","); - buffer.append(scoreThreshold); - buffer.append(","); + buffer.append(targetNumHits).append(","); + buffer.append(scoreThreshold).append(","); buffer.append(thresholdBoostFactor); - buffer.append(")"); - buffer.append(" "); + buffer.append(") "); } @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java index 4fa2ed8b214..e8817a44133 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java @@ -18,16 +18,20 @@ import java.nio.ByteBuffer; */ public final class WeakAndItem extends NonReducibleCompositeItem { + /** The default N used if none is specified: 100 */ + public static final int defaultN = 100; + private int n; private String index; private int scoreThreshold = 0; - public ItemType getItemType() { - return ItemType.WEAK_AND; + /** Creates a WAND item with default N */ + public WeakAndItem() { + this(defaultN); } - public String getName() { - return "WAND"; + public WeakAndItem(int N) { + this("", N); } /** @@ -42,50 +46,37 @@ public final class WeakAndItem extends NonReducibleCompositeItem { this.n = n; this.index = (index == null) ? "" : index; } - public WeakAndItem(int N) { - this("", N); - } - /** Sets the index name of all subitems of this */ + @Override + public ItemType getItemType() { return ItemType.WEAK_AND; } + + @Override + public String getName() { return "WEAKAND"; } + + @Override public void setIndexName(String index) { String toSet = (index == null) ? "" : index; super.setIndexName(toSet); this.index = toSet; } - public String getIndexName() { - return index; - } + public String getIndexName() { return index; } /** Appends the heading of this string - <code>[getName()]([limit]) </code> */ + @Override protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); buffer.append(n); - buffer.append(")"); - buffer.append(" "); + buffer.append(") "); } - /** The default N used if none is specified: 100 */ - public static final int defaultN = 100; + public int getN() { return n; } - /** Creates a WAND item with default N */ - public WeakAndItem() { - this(defaultN); - } - - public int getN() { - return n; - } - - public void setN(int N) { - this.n = N; - } + public void setN(int N) { this.n = N; } @Deprecated // TODO: Remove on Vespa 8 - public int getScoreThreshold() { - return scoreThreshold; - } + public int getScoreThreshold() { return scoreThreshold; } /** * Noop. @@ -93,9 +84,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { * @deprecated has no effect */ @Deprecated // TODO: Remove on Vespa 8 - public void setScoreThreshold(int scoreThreshold) { - this.scoreThreshold = scoreThreshold; - } + public void setScoreThreshold(int scoreThreshold) { this.scoreThreshold = scoreThreshold; } @Override protected void encodeThis(ByteBuffer buffer) { @@ -111,9 +100,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { } @Override - public int hashCode() { - return super.hashCode() + 31 * n; - } + public int hashCode() { return super.hashCode() + 31 * n; } /** Returns whether this item is of the same class and contains the same state as the given item. */ @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java index 74a993b0413..8b878417912 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java @@ -147,10 +147,10 @@ public class AdvancedParser extends StructuredParser { return equiv; } return topLevelItem; - } else if (isTheWord("wand", item)) { + } else if (isTheWord("wand", item) || isTheWord("weakand", item)) { int n = consumeNumericArgument(); if (n == 0) - n=WeakAndItem.defaultN; + n = WeakAndItem.defaultN; if (topLevelIsClosed || !(topLevelItem instanceof WeakAndItem) || n != ((WeakAndItem)topLevelItem).getN()) { WeakAndItem wand = new WeakAndItem(); wand.setN(n); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index dcf052d28e6..1bcb640e3a5 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -79,7 +79,7 @@ public abstract class InvokerFactory { success.add(node); } } - if ( ! searchCluster.isPartialGroupCoverageSufficient(groupId, success) && !acceptIncompleteCoverage) { + if ( ! searchCluster.isPartialGroupCoverageSufficient(success) && !acceptIncompleteCoverage) { return Optional.empty(); } if (invokers.size() == 0) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java index e5066797b06..dca5892e0e7 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java @@ -92,7 +92,7 @@ public class Group { } @Override - public String toString() { return "search group " + id; } + public String toString() { return "group " + id; } @Override public int hashCode() { return id; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index 8f465070de4..9807a978647 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -71,7 +71,7 @@ public class Node { } /** Updates the active documents on this node */ - void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); } + public void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); } /** Returns the active documents on this node. If unknown, 0 is returned. */ long getActiveDocuments() { return activeDocuments.get(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index b3b2c23e7dc..421082bb5dc 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; +import com.google.common.math.Quantiles; import com.yahoo.container.handler.VipStatus; import com.yahoo.net.HostName; import com.yahoo.prelude.Pong; @@ -13,6 +14,8 @@ import com.yahoo.search.cluster.NodeManager; import com.yahoo.search.dispatch.TopKEstimator; import com.yahoo.vespa.config.search.DispatchConfig; +import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -311,33 +314,33 @@ public class SearchCluster implements NodeManager<Node> { boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), group.getActiveDocuments(), group.getActiveDocuments()); - trackGroupCoverageChanges(0, group, sufficientCoverage, group.getActiveDocuments()); + trackGroupCoverageChanges(group, sufficientCoverage, group.getActiveDocuments()); } private void pingIterationCompletedMultipleGroups() { - int numGroups = orderedGroups().size(); - // Update active documents per group and use it to decide if the group should be active - long[] activeDocumentsInGroup = new long[numGroups]; - long sumOfActiveDocuments = 0; - for(int i = 0; i < numGroups; i++) { - Group group = orderedGroups().get(i); - group.aggregateNodeValues(); - activeDocumentsInGroup[i] = group.getActiveDocuments(); - sumOfActiveDocuments += activeDocumentsInGroup[i]; - } - + aggregateNodeValues(); + long medianDocuments = medianDocumentsPerGroup(); boolean anyGroupsSufficientCoverage = false; - for (int i = 0; i < numGroups; i++) { - Group group = orderedGroups().get(i); - long activeDocuments = activeDocumentsInGroup[i]; - long averageDocumentsInOtherGroups = (sumOfActiveDocuments - activeDocuments) / (numGroups - 1); - boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), activeDocuments, averageDocumentsInOtherGroups); + for (Group group : orderedGroups()) { + boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), + group.getActiveDocuments(), + medianDocuments); anyGroupsSufficientCoverage = anyGroupsSufficientCoverage || sufficientCoverage; updateSufficientCoverage(group, sufficientCoverage); - trackGroupCoverageChanges(i, group, sufficientCoverage, averageDocumentsInOtherGroups); + trackGroupCoverageChanges(group, sufficientCoverage, medianDocuments); } } + private void aggregateNodeValues() { + orderedGroups().forEach(Group::aggregateNodeValues); + } + + private long medianDocumentsPerGroup() { + if (orderedGroups().isEmpty()) return 0; + var activeDocuments = orderedGroups().stream().map(Group::getActiveDocuments).collect(Collectors.toList()); + return (long)Quantiles.median().compute(activeDocuments); + } + /** * Update statistics after a round of issuing pings. * Note that this doesn't wait for pings to return, so it will typically accumulate data from @@ -353,10 +356,10 @@ public class SearchCluster implements NodeManager<Node> { } } - private boolean isGroupCoverageSufficient(int workingNodesInGroup, long activeDocuments, long averageDocumentsInOtherGroups) { - double documentCoverage = 100.0 * (double) activeDocuments / averageDocumentsInOtherGroups; + private boolean isGroupCoverageSufficient(int workingNodesInGroup, long activeDocuments, long medianDocuments) { + double documentCoverage = 100.0 * (double) activeDocuments / medianDocuments; - if (averageDocumentsInOtherGroups > 0 && documentCoverage < dispatchConfig.minActivedocsPercentage()) + if (medianDocuments > 0 && documentCoverage < dispatchConfig.minActivedocsPercentage()) return false; if ( ! isGroupNodeCoverageSufficient(workingNodesInGroup)) @@ -380,45 +383,22 @@ public class SearchCluster implements NodeManager<Node> { /** * Calculate whether a subset of nodes in a group has enough coverage */ - public boolean isPartialGroupCoverageSufficient(OptionalInt knownGroupId, List<Node> nodes) { - if (orderedGroups().size() == 1) { - boolean sufficient = nodes.size() >= wantedGroupSize() - dispatchConfig.maxNodesDownPerGroup(); - return sufficient; - } - - if (knownGroupId.isEmpty()) { - return false; - } - int groupId = knownGroupId.getAsInt(); - Group group = groups().get(groupId); - if (group == null) { - return false; - } - long sumOfActiveDocuments = 0; - int otherGroups = 0; - for (Group g : orderedGroups()) { - if (g.id() != groupId) { - sumOfActiveDocuments += g.getActiveDocuments(); - otherGroups++; - } - } - long activeDocuments = 0; - for (Node n : nodes) { - activeDocuments += n.getActiveDocuments(); - } - long averageDocumentsInOtherGroups = sumOfActiveDocuments / otherGroups; - return isGroupCoverageSufficient(nodes.size(), activeDocuments, averageDocumentsInOtherGroups); + public boolean isPartialGroupCoverageSufficient(List<Node> nodes) { + if (orderedGroups().size() == 1) + return nodes.size() >= wantedGroupSize() - dispatchConfig.maxNodesDownPerGroup(); + long activeDocuments = nodes.stream().mapToLong(Node::getActiveDocuments).sum(); + return isGroupCoverageSufficient(nodes.size(), activeDocuments, medianDocumentsPerGroup()); } - private void trackGroupCoverageChanges(int index, Group group, boolean fullCoverage, long averageDocuments) { + private void trackGroupCoverageChanges(Group group, boolean fullCoverage, long medianDocuments) { if ( ! hasInformationAboutAllNodes()) return; // Be silent until we know what we are talking about. boolean changed = group.isFullCoverageStatusChanged(fullCoverage); if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { nextLogTime = System.currentTimeMillis() + 30 * 1000; int requiredNodes = group.nodes().size() - dispatchConfig.maxNodesDownPerGroup(); if (fullCoverage) { - log.info(() -> String.format("Cluster %s: Group %d is now good again (%d/%d active docs, coverage %d/%d)", - clusterId, index, group.getActiveDocuments(), averageDocuments, + log.info(() -> String.format("Cluster %s: %s is now good again (%d/%d active docs, coverage %d/%d)", + clusterId, group, group.getActiveDocuments(), medianDocuments, group.workingNodes(), group.nodes().size())); } else { StringBuilder missing = new StringBuilder(); @@ -427,9 +407,9 @@ public class SearchCluster implements NodeManager<Node> { missing.append('\n').append(node); } } - log.warning(() -> String.format("Cluster %s: Coverage of group %d is only %d/%d (requires %d) (%d/%d active docs) Failed nodes are:%s", - clusterId, index, group.workingNodes(), group.nodes().size(), requiredNodes, - group.getActiveDocuments(), averageDocuments, missing)); + log.warning(() -> String.format("Cluster %s: Coverage of %s is only %d/%d (requires %d) (%d/%d active docs) Failed nodes are:%s", + clusterId, group, group.workingNodes(), group.nodes().size(), requiredNodes, + group.getActiveDocuments(), medianDocuments, missing)); } } } diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index bcd5822a6f6..d3f07bae428 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -14,76 +14,46 @@ import com.yahoo.search.yql.yqlplusParser.AnnotateExpressionContext; import com.yahoo.search.yql.yqlplusParser.ArgumentContext; import com.yahoo.search.yql.yqlplusParser.ArgumentsContext; import com.yahoo.search.yql.yqlplusParser.ArrayLiteralContext; -import com.yahoo.search.yql.yqlplusParser.ArrayTypeContext; import com.yahoo.search.yql.yqlplusParser.Call_sourceContext; import com.yahoo.search.yql.yqlplusParser.ConstantArrayContext; import com.yahoo.search.yql.yqlplusParser.ConstantExpressionContext; import com.yahoo.search.yql.yqlplusParser.ConstantMapExpressionContext; import com.yahoo.search.yql.yqlplusParser.ConstantPropertyNameAndValueContext; -import com.yahoo.search.yql.yqlplusParser.Delete_statementContext; import com.yahoo.search.yql.yqlplusParser.DereferencedExpressionContext; import com.yahoo.search.yql.yqlplusParser.EqualityExpressionContext; import com.yahoo.search.yql.yqlplusParser.ExpressionContext; -import com.yahoo.search.yql.yqlplusParser.FallbackContext; import com.yahoo.search.yql.yqlplusParser.Field_defContext; -import com.yahoo.search.yql.yqlplusParser.Field_names_specContext; -import com.yahoo.search.yql.yqlplusParser.Field_values_group_specContext; -import com.yahoo.search.yql.yqlplusParser.Field_values_specContext; import com.yahoo.search.yql.yqlplusParser.IdentContext; -import com.yahoo.search.yql.yqlplusParser.Import_listContext; -import com.yahoo.search.yql.yqlplusParser.Import_statementContext; import com.yahoo.search.yql.yqlplusParser.InNotInTargetContext; -import com.yahoo.search.yql.yqlplusParser.Insert_sourceContext; -import com.yahoo.search.yql.yqlplusParser.Insert_statementContext; -import com.yahoo.search.yql.yqlplusParser.Insert_valuesContext; -import com.yahoo.search.yql.yqlplusParser.JoinExpressionContext; -import com.yahoo.search.yql.yqlplusParser.Join_exprContext; import com.yahoo.search.yql.yqlplusParser.LimitContext; import com.yahoo.search.yql.yqlplusParser.Literal_elementContext; import com.yahoo.search.yql.yqlplusParser.Literal_listContext; import com.yahoo.search.yql.yqlplusParser.LogicalANDExpressionContext; import com.yahoo.search.yql.yqlplusParser.LogicalORExpressionContext; import com.yahoo.search.yql.yqlplusParser.MapExpressionContext; -import com.yahoo.search.yql.yqlplusParser.MapTypeContext; -import com.yahoo.search.yql.yqlplusParser.Merge_componentContext; -import com.yahoo.search.yql.yqlplusParser.Merge_statementContext; -import com.yahoo.search.yql.yqlplusParser.ModuleIdContext; -import com.yahoo.search.yql.yqlplusParser.ModuleNameContext; import com.yahoo.search.yql.yqlplusParser.MultiplicativeExpressionContext; import com.yahoo.search.yql.yqlplusParser.Namespaced_nameContext; -import com.yahoo.search.yql.yqlplusParser.Next_statementContext; import com.yahoo.search.yql.yqlplusParser.OffsetContext; import com.yahoo.search.yql.yqlplusParser.OrderbyContext; import com.yahoo.search.yql.yqlplusParser.Orderby_fieldContext; import com.yahoo.search.yql.yqlplusParser.Output_specContext; -import com.yahoo.search.yql.yqlplusParser.Paged_clauseContext; -import com.yahoo.search.yql.yqlplusParser.ParamsContext; import com.yahoo.search.yql.yqlplusParser.Pipeline_stepContext; -import com.yahoo.search.yql.yqlplusParser.Procedure_argumentContext; -import com.yahoo.search.yql.yqlplusParser.Program_arglistContext; import com.yahoo.search.yql.yqlplusParser.Project_specContext; import com.yahoo.search.yql.yqlplusParser.ProgramContext; import com.yahoo.search.yql.yqlplusParser.PropertyNameAndValueContext; import com.yahoo.search.yql.yqlplusParser.Query_statementContext; import com.yahoo.search.yql.yqlplusParser.RelationalExpressionContext; import com.yahoo.search.yql.yqlplusParser.RelationalOpContext; -import com.yahoo.search.yql.yqlplusParser.Returning_specContext; import com.yahoo.search.yql.yqlplusParser.Scalar_literalContext; -import com.yahoo.search.yql.yqlplusParser.Select_source_joinContext; import com.yahoo.search.yql.yqlplusParser.Select_source_multiContext; import com.yahoo.search.yql.yqlplusParser.Select_statementContext; -import com.yahoo.search.yql.yqlplusParser.Selectvar_statementContext; import com.yahoo.search.yql.yqlplusParser.Sequence_sourceContext; import com.yahoo.search.yql.yqlplusParser.Source_listContext; import com.yahoo.search.yql.yqlplusParser.Source_specContext; import com.yahoo.search.yql.yqlplusParser.Source_statementContext; import com.yahoo.search.yql.yqlplusParser.StatementContext; import com.yahoo.search.yql.yqlplusParser.TimeoutContext; -import com.yahoo.search.yql.yqlplusParser.TypenameContext; import com.yahoo.search.yql.yqlplusParser.UnaryExpressionContext; -import com.yahoo.search.yql.yqlplusParser.Update_statementContext; -import com.yahoo.search.yql.yqlplusParser.Update_valuesContext; -import com.yahoo.search.yql.yqlplusParser.ViewContext; import com.yahoo.search.yql.yqlplusParser.WhereContext; import org.antlr.v4.runtime.BaseErrorListener; @@ -126,7 +96,6 @@ final class ProgramParser { return prepareParser(file.getAbsoluteFile().toString(), new CaseInsensitiveFileStream(file.getAbsolutePath())); } - private yqlplusParser prepareParser(String programName, CharStream input) { yqlplusLexer lexer = new yqlplusLexer(input); lexer.removeErrorListeners(); @@ -168,41 +137,18 @@ final class ProgramParser { try { return parser.program(); } catch (RecognitionException e) { - //Retry parsing using full LL mode + // Retry parsing using full LL mode parser.reset(); parser.getInterpreter().setPredictionMode(PredictionMode.LL); return parser.program(); } } - public OperatorNode<StatementOperator> parse(String programName, InputStream program) throws IOException, RecognitionException { - yqlplusParser parser = prepareParser(programName, program); - return convertProgram(parseProgram(parser), parser, programName); - } - public OperatorNode<StatementOperator> parse(String programName, String program) throws IOException, RecognitionException { yqlplusParser parser = prepareParser(programName, program); return convertProgram(parseProgram(parser), parser, programName); } - public OperatorNode<StatementOperator> parse(File input) throws IOException, RecognitionException { - yqlplusParser parser = prepareParser(input); - return convertProgram(parseProgram(parser), parser, input.getAbsoluteFile().toString()); - } - - public OperatorNode<ExpressionOperator> parseExpression(String input) throws IOException, RecognitionException { - return convertExpr(prepareParser("<expression>", input).expression(false).getRuleContext(), new Scope()); - } - - public OperatorNode<ExpressionOperator> parseExpression(String input, Set<String> visibleAliases) throws IOException, RecognitionException { - Scope scope = new Scope(); - final Location loc = new Location("<expression>", -1, -1); - for (String alias : visibleAliases) { - scope.defineDataSource(loc, alias); - } - return convertExpr(prepareParser("<expression>", input).expression(false).getRuleContext(), scope); - } - private Location toLocation(Scope scope, ParseTree node) { Token start; if (node instanceof ParserRuleContext) { @@ -212,8 +158,7 @@ final class ProgramParser { } else { throw new ProgramCompileException("Location is not available for type " + node.getClass()); } - Location location = new Location(scope != null? scope.programName: "<string>", start.getLine(), start.getCharPositionInLine()); - return location; + return new Location(scope != null? scope.programName: "<string>", start.getLine(), start.getCharPositionInLine()); } private List<String> readName(Namespaced_nameContext node) { @@ -230,14 +175,6 @@ final class ProgramParser { private final List<String> binding; - Binding(String moduleName, String exportName) { - this.binding = ImmutableList.of(moduleName, exportName); - } - - Binding(String moduleName) { - this.binding = ImmutableList.of(moduleName); - } - Binding(List<String> binding) { this.binding = binding; } @@ -263,13 +200,6 @@ final class ProgramParser { final yqlplusParser parser; final String programName; - Scope() { - this.parser = null; - this.programName = null; - this.root = this; - this.parent = null; - } - Scope(yqlplusParser parser, String programName) { this.parser = parser; this.programName = programName; @@ -288,15 +218,10 @@ final class ProgramParser { return parser; } - public String getProgramName() { - return programName; - } - public Set<String> getCursors() { return cursors; } - boolean isBound(String name) { // bindings live only in the 'root' node return root.bindings.containsKey(name); @@ -329,13 +254,6 @@ final class ProgramParser { root.bindings.put(symbolName, new Binding(binding)); } - public void bindModuleSymbol(Location loc, List<String> moduleName, String exportName, String symbolName) { - ImmutableList.Builder<String> builder = ImmutableList.builder(); - builder.addAll(moduleName); - builder.add(exportName); - bindModule(loc, builder.build(), symbolName); - } - public void defineDataSource(Location loc, String name) { if (isCursor(name)) { throw new ProgramCompileException(loc, "Alias '%s' is already used.", name); @@ -376,13 +294,12 @@ final class ProgramParser { } } - private OperatorNode<SequenceOperator> convertSelectOrInsertOrUpdateOrDelete(ParseTree node, Scope scopeParent) { + private OperatorNode<SequenceOperator> convertSelect(ParseTree node, Scope scopeParent) { - Preconditions.checkArgument(node instanceof Select_statementContext || node instanceof Insert_statementContext || - node instanceof Update_statementContext || node instanceof Delete_statementContext); + Preconditions.checkArgument(node instanceof Select_statementContext); - // SELECT^ select_field_spec select_source where? orderby? limit? offset? timeout? fallback? - // select is the only place to define where/orderby/limit/offset and joins + // SELECT^ select_field_spec select_source where? orderby? limit? offset? timeout? + // select is the only place to define where/orderby/limit/offset Scope scope = scopeParent.child(); ProjectionBuilder proj = null; OperatorNode<SequenceOperator> source = null; @@ -391,29 +308,18 @@ final class ProgramParser { OperatorNode<ExpressionOperator> offset = null; OperatorNode<ExpressionOperator> limit = null; OperatorNode<ExpressionOperator> timeout = null; - OperatorNode<SequenceOperator> fallback = null; - OperatorNode<SequenceOperator> insertValues = null; - OperatorNode<ExpressionOperator> updateValues = null; - ParseTree sourceNode; - - if (node instanceof Select_statementContext ) { - sourceNode = node.getChild(2) != null ? node.getChild(2).getChild(0):null; - } else { - sourceNode = node.getChild(1); - } + ParseTree sourceNode = node.getChild(2) != null ? node.getChild(2).getChild(0):null; if (sourceNode != null) { switch (getParseTreeIndex(sourceNode)) { // ALL_SOURCE and MULTI_SOURCE are how FROM SOURCES // *|source_name,... are parsed - // They can't be used directly with the JOIN syntax at this time - case yqlplusParser.RULE_select_source_all: { + case yqlplusParser.RULE_select_source_all: Location location = toLocation(scope, sourceNode.getChild(2)); source = OperatorNode.create(location, SequenceOperator.ALL); source.putAnnotation("alias", "row"); scope.defineDataSource(location, "row"); - } break; case yqlplusParser.RULE_select_source_multi: Source_listContext multiSourceContext = ((Select_source_multiContext) sourceNode).source_list(); @@ -421,22 +327,8 @@ final class ProgramParser { source.putAnnotation("alias", "row"); scope.defineDataSource(toLocation(scope, multiSourceContext), "row"); break; - case yqlplusParser.RULE_select_source_join: + case yqlplusParser.RULE_select_source_from: source = convertSource((ParserRuleContext) sourceNode.getChild(1), scope); - List<Join_exprContext> joinContexts = ((Select_source_joinContext)sourceNode).join_expr(); - for (Join_exprContext joinContext:joinContexts) { - source = convertJoin(joinContext, source, scope); - } - break; - case yqlplusParser.RULE_insert_source: - Insert_sourceContext insertSourceContext = (Insert_sourceContext) sourceNode; - source = convertSource((ParserRuleContext)insertSourceContext.getChild(1), scope); - break; - case yqlplusParser.RULE_delete_source: - source = convertSource((ParserRuleContext)sourceNode.getChild(1), scope); - break; - case yqlplusParser.RULE_update_source: - source = convertSource((ParserRuleContext)sourceNode.getChild(0), scope); break; } } else { @@ -451,9 +343,6 @@ final class ProgramParser { proj = readProjection(((Project_specContext) child.getChild(0)).field_def(), scope); } break; - case yqlplusParser.RULE_returning_spec: - proj = readProjection(((Returning_specContext) child).select_field_spec().project_spec().field_def(), scope); - break; case yqlplusParser.RULE_where: filter = convertExpr(((WhereContext) child).expression(), scope); break; @@ -475,23 +364,6 @@ final class ProgramParser { case yqlplusParser.RULE_timeout: timeout = convertExpr(((TimeoutContext) child).fixed_or_parameter(), scope); break; - case yqlplusParser.RULE_fallback: - fallback = convertQuery(((FallbackContext) child).select_statement(), scope); - break; - case yqlplusParser.RULE_insert_values: - if (child.getChild(0) instanceof yqlplusParser.Query_statementContext) { - insertValues = convertQuery(child.getChild(0).getChild(0), scope); - } else { - insertValues = readBatchValues(((Insert_valuesContext) child).field_names_spec(), ((Insert_valuesContext)child).field_values_group_spec(), scope); - } - break; - case yqlplusParser.RULE_update_values: - if (getParseTreeIndex(child.getChild(0)) == yqlplusParser.RULE_field_def) { - updateValues = readValues(((Update_valuesContext)child).field_def(), scope); - } else { - updateValues = readValues((Field_names_specContext)child.getChild(0), (Field_values_specContext)child.getChild(2), scope); - } - break; } } // now assemble the logical plan @@ -500,26 +372,6 @@ final class ProgramParser { if (filter != null) { result = OperatorNode.create(SequenceOperator.FILTER, result, filter); } - // insert values - if (insertValues != null) { - result = OperatorNode.create(SequenceOperator.INSERT, result, insertValues); - } - // update - if (updateValues != null) { - if (filter != null) { - result = OperatorNode.create(SequenceOperator.UPDATE, source, updateValues, filter); - } else { - result = OperatorNode.create(SequenceOperator.UPDATE_ALL, source, updateValues); - } - } - // delete - if (getParseTreeIndex(node) == yqlplusParser.RULE_delete_statement) { - if (filter != null) { - result = OperatorNode.create(SequenceOperator.DELETE, source, filter); - } else { - result = OperatorNode.create(SequenceOperator.DELETE_ALL, source); - } - } // then sort (or project and sort) boolean projectBeforeSort = false; if (orderby != null) { @@ -558,30 +410,9 @@ final class ProgramParser { if (timeout != null) { result = OperatorNode.create(SequenceOperator.TIMEOUT, result, timeout); } - // if there's a fallback, emit a fallback node - if (fallback != null) { - result = OperatorNode.create(SequenceOperator.FALLBACK, result, fallback); - } return result; } - private OperatorNode<ExpressionOperator> readValues(List<Field_defContext> fieldDefs, Scope scope) { - List<String> fieldNames; - List<OperatorNode<ExpressionOperator>> fieldValues; - int numPairs = fieldDefs.size(); - fieldNames = Lists.newArrayListWithExpectedSize(numPairs); - fieldValues = Lists.newArrayListWithExpectedSize(numPairs); - for (int j = 0; j < numPairs; j++) { - ParseTree startNode = fieldDefs.get(j); - while(startNode.getChildCount() < 3) { - startNode = startNode.getChild(0); - } - fieldNames.add((String) convertExpr(startNode.getChild(0), scope).getArgument(1)); - fieldValues.add(convertExpr(startNode.getChild(2), scope)); - } - return OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues); - } - private OperatorNode<SequenceOperator> readMultiSource(Scope scope, Source_listContext multiSource) { List<List<String>> sourceNameList = Lists.newArrayList(); List<Namespaced_nameContext> nameSpaces = multiSource.namespaced_name(); @@ -591,9 +422,7 @@ final class ProgramParser { } return OperatorNode.create(toLocation(scope, multiSource), SequenceOperator.MULTISOURCE, sourceNameList); } -// pipeline_step -// : namespaced_name arguments[false]? -// ; + private OperatorNode<SequenceOperator> convertPipe(Query_statementContext queryStatementContext, List<Pipeline_stepContext> nodes, Scope scope) { OperatorNode<SequenceOperator> result = convertQuery(queryStatementContext.getChild(0), scope.getRoot()); for (Pipeline_stepContext step:nodes) { @@ -603,7 +432,7 @@ final class ProgramParser { } else { List<String> name = readName(step.namespaced_name()); List<OperatorNode<ExpressionOperator>> args = ImmutableList.of(); - //LPAREN (argument[$in_select] (COMMA argument[$in_select])*) RPAREN + // LPAREN (argument[$in_select] (COMMA argument[$in_select])*) RPAREN if (step.getChildCount() > 1) { ArgumentsContext arguments = step.arguments(); if (arguments.getChildCount() > 2) { @@ -621,56 +450,25 @@ final class ProgramParser { return result; } - private OperatorNode<SequenceOperator> convertMerge(List<Merge_componentContext> mergeComponentList, Scope scope) { - Preconditions.checkArgument(mergeComponentList != null); - List<OperatorNode<SequenceOperator>> sources = Lists.newArrayListWithExpectedSize(mergeComponentList.size()); - for (Merge_componentContext mergeComponent:mergeComponentList) { - Select_statementContext selectContext = mergeComponent.select_statement(); - Source_statementContext sourceContext = mergeComponent.source_statement(); - if (selectContext != null) { - sources.add(convertQuery(selectContext, scope.getRoot())); - } else { - sources.add(convertQuery(sourceContext, scope.getRoot())); - } - } - return OperatorNode.create(SequenceOperator.MERGE, sources); - } - private OperatorNode<SequenceOperator> convertQuery(ParseTree node, Scope scope) { - if (node instanceof Select_statementContext - || node instanceof Insert_statementContext - || node instanceof Update_statementContext - || node instanceof Delete_statementContext) { - return convertSelectOrInsertOrUpdateOrDelete(node, scope.getRoot()); + if (node instanceof Select_statementContext) { + return convertSelect(node, scope.getRoot()); } else if (node instanceof Source_statementContext) { // for pipe Source_statementContext sourceStatementContext = (Source_statementContext)node; return convertPipe(sourceStatementContext.query_statement(), sourceStatementContext.pipeline_step(), scope); - } else if (node instanceof Merge_statementContext) { - return convertMerge(((Merge_statementContext)node).merge_component(), scope); } else { throw new IllegalArgumentException("Unexpected argument type to convertQueryStatement: " + node.toStringTree()); } } - private OperatorNode<SequenceOperator> convertJoin(Join_exprContext node, OperatorNode<SequenceOperator> left, Scope scope) { - Source_specContext sourceSpec = node.source_spec(); - OperatorNode<SequenceOperator> right = convertSource(sourceSpec, scope); - JoinExpressionContext joinContext = node.joinExpression(); - OperatorNode<ExpressionOperator> joinExpression = readBinOp(ExpressionOperator.valueOf("EQ"), joinContext.getChild(0), joinContext.getChild(2), scope); - if (joinExpression.getOperator() != ExpressionOperator.EQ) { - throw new ProgramCompileException(joinExpression.getLocation(), "Unexpected join expression type: %s (expected EQ)", joinExpression.getOperator()); - } - return OperatorNode.create(toLocation(scope, sourceSpec), node.join_spec().LEFT() != null ? SequenceOperator.LEFT_JOIN : SequenceOperator.JOIN, left, right, joinExpression); - } - private String assignAlias(String alias, ParserRuleContext node, Scope scope) { if (alias == null) { alias = "source"; } - if (node != null && node instanceof yqlplusParser.Alias_defContext) { - //alias_def : (AS? ID); + if (node instanceof yqlplusParser.Alias_defContext) { + // alias_def : (AS? ID); ParseTree idChild = node; if (node.getChildCount() > 1) { idChild = node.getChild(1); @@ -690,20 +488,14 @@ final class ProgramParser { scope.defineDataSource(null, candidate); return alias; } - } - - private OperatorNode<SequenceOperator> convertSource(ParserRuleContext sourceSpecNode, Scope scope) { + } + private OperatorNode<SequenceOperator> convertSource(ParserRuleContext sourceSpecNode, Scope scope) { // DataSources String alias; OperatorNode<SequenceOperator> result; ParserRuleContext dataSourceNode = sourceSpecNode; ParserRuleContext aliasContext = null; - //data_source - //: call_source - //| LPAREN source_statement RPAREN - //| sequence_source - //; if (sourceSpecNode instanceof Source_specContext) { dataSourceNode = (ParserRuleContext)sourceSpecNode.getChild(0); if (sourceSpecNode.getChildCount() == 2) { @@ -717,7 +509,6 @@ final class ProgramParser { } } switch (getParseTreeIndex(dataSourceNode)) { - case yqlplusParser.RULE_write_data_source: case yqlplusParser.RULE_call_source: { List<String> names = readName(dataSourceNode.getChild(Namespaced_nameContext.class, 0)); alias = assignAlias(names.get(names.size() - 1), aliasContext, scope); @@ -763,60 +554,9 @@ final class ProgramParser { return result; } - private OperatorNode<TypeOperator> decodeType(Scope scope, TypenameContext type) { - - TypeOperator op; - ParseTree typeNode = type.getChild(0); - switch (getParseTreeIndex(typeNode)) { - case yqlplusParser.TYPE_BOOLEAN: - op = TypeOperator.BOOLEAN; - break; - case yqlplusParser.TYPE_BYTE: - op = TypeOperator.BYTE; - break; - case yqlplusParser.TYPE_DOUBLE: - op = TypeOperator.DOUBLE; - break; - case yqlplusParser.TYPE_INT16: - op = TypeOperator.INT16; - break; - case yqlplusParser.TYPE_INT32: - op = TypeOperator.INT32; - break; - case yqlplusParser.TYPE_INT64: - op = TypeOperator.INT64; - break; - case yqlplusParser.TYPE_STRING: - op = TypeOperator.STRING; - break; - case yqlplusParser.TYPE_TIMESTAMP: - op = TypeOperator.TIMESTAMP; - break; - case yqlplusParser.RULE_arrayType: - return OperatorNode.create(toLocation(scope, typeNode), TypeOperator.ARRAY, decodeType(scope, ((ArrayTypeContext)typeNode).getChild(TypenameContext.class, 0))); - case yqlplusParser.RULE_mapType: - return OperatorNode.create(toLocation(scope, typeNode), TypeOperator.MAP, decodeType(scope, ((MapTypeContext)typeNode).getChild(TypenameContext.class, 0))); - default: - throw new ProgramCompileException("Unknown type " + typeNode.getText()); - } - return OperatorNode.create(toLocation(scope, typeNode), op); - } - - private List<String> createBindingName(ParseTree node) { - if (node instanceof ModuleNameContext) { - if (((ModuleNameContext)node).namespaced_name() != null) { - return readName(((ModuleNameContext)node).namespaced_name()); - } else if (((ModuleNameContext)node).literalString() != null) { - return ImmutableList.of(((ModuleNameContext)node).literalString().STRING().getText()); - } - } else if (node instanceof ModuleIdContext) { - return ImmutableList.of(node.getText()); - } - throw new ProgramCompileException("Wrong context"); - } - - private OperatorNode<StatementOperator> convertProgram( - ParserRuleContext program, yqlplusParser parser, String programName) { + private OperatorNode<StatementOperator> convertProgram(ParserRuleContext program, + yqlplusParser parser, + String programName) { Scope scope = new Scope(parser, programName); List<OperatorNode<StatementOperator>> stmts = Lists.newArrayList(); int output = 0; @@ -825,148 +565,37 @@ final class ProgramParser { continue; } ParserRuleContext ruleContext = (ParserRuleContext) node; - switch (ruleContext.getRuleIndex()) { - case yqlplusParser.RULE_params: { - // ^(ARGUMENT ident typeref expression?) - ParamsContext paramsContext = (ParamsContext) ruleContext; - Program_arglistContext program_arglistContext = paramsContext.program_arglist(); - if (program_arglistContext != null) { - List<Procedure_argumentContext> argList = program_arglistContext.procedure_argument(); - for (Procedure_argumentContext procedureArgumentContext : argList) { - String name = procedureArgumentContext.ident().getText(); - OperatorNode<TypeOperator> type = decodeType(scope, procedureArgumentContext.getChild(TypenameContext.class, 0)); - OperatorNode<ExpressionOperator> defaultValue = OperatorNode.create(ExpressionOperator.NULL); - if (procedureArgumentContext.expression() != null) { - defaultValue = convertExpr(procedureArgumentContext.expression(), scope); - } - scope.defineVariable(toLocation(scope, procedureArgumentContext), name); - stmts.add(OperatorNode.create(StatementOperator.ARGUMENT, name, type, defaultValue)); - } - } - break; - } - case yqlplusParser.RULE_import_statement: { - Import_statementContext importContext = (Import_statementContext) ruleContext; - if (null == importContext.import_list()) { - List<String> name = createBindingName(node.getChild(1)); - String target; - Location location = toLocation(scope, node.getChild(1)); - if (node.getChildCount() == 2) { - target = name.get(0); - } else if (node.getChildCount() == 4) { - target = node.getChild(3).getText(); - } else { - throw new ProgramCompileException("Unknown node count for IMPORT: " + node.toStringTree()); - } - scope.bindModule(location, name, target); - } else { - // | FROM moduleName IMPORT import_list -> ^(IMPORT_FROM - // moduleName import_list+) - Import_listContext importListContext = importContext.import_list(); - List<String> name = createBindingName(importContext.moduleName()); - Location location = toLocation(scope, importContext.moduleName()); - List<ModuleIdContext> moduleIds = importListContext.moduleId(); - List<String> symbols = Lists.newArrayListWithExpectedSize(moduleIds.size()); - for (ModuleIdContext cnode : moduleIds) { - symbols.add(cnode.ID().getText()); - } - for (String sym : symbols) { - scope.bindModuleSymbol(location, name, sym, sym); - } - } - break; - } + if (ruleContext.getRuleIndex() != yqlplusParser.RULE_statement) + throw new ProgramCompileException("Unknown program element: " + node.getText()); - // DDL - case yqlplusParser.RULE_ddl: - ruleContext = (ParserRuleContext)ruleContext.getChild(0); - break; - case yqlplusParser.RULE_view: { - // view and projection expansion now has to be done by the - // execution engine - // since views/projections, in order to be useful, have to - // support being used from outside the same program - ViewContext viewContext = (ViewContext) ruleContext; - Location loc = toLocation(scope, viewContext); - scope.getRoot().defineView(loc, viewContext.ID().getText()); - stmts.add(OperatorNode.create(loc, StatementOperator.DEFINE_VIEW, viewContext.ID().getText(), convertQuery(viewContext.source_statement(), scope.getRoot()))); - break; + // ^(STATEMENT_QUERY source_statement paged_clause? output_spec?) + StatementContext statementContext = (StatementContext) ruleContext; + Source_statementContext source_statement = statementContext.output_statement().source_statement(); + OperatorNode<SequenceOperator> query; + if (source_statement.getChildCount() == 1) { + query = convertQuery( source_statement.query_statement().getChild(0), scope); + } else { + query = convertQuery(source_statement, scope); } - case yqlplusParser.RULE_statement: { - // ^(STATEMENT_QUERY source_statement paged_clause? - // output_spec?) - StatementContext statementContext = (StatementContext) ruleContext; - switch (getParseTreeIndex(ruleContext.getChild(0))) { - case yqlplusParser.RULE_selectvar_statement: { - // ^(STATEMENT_SELECTVAR ident source_statement) - Selectvar_statementContext selectVarContext = (Selectvar_statementContext) ruleContext.getChild(0); - String variable = selectVarContext.ident().getText(); - OperatorNode<SequenceOperator> query = convertQuery(selectVarContext.source_statement(), scope); - Location location = toLocation(scope, selectVarContext.ident()); - scope.defineVariable(location, variable); - stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable)); - break; - } - case yqlplusParser.RULE_next_statement: { - // NEXT^ literalString OUTPUT! AS! ident - Next_statementContext nextStateContext = (Next_statementContext) ruleContext.getChild(0); - String continuationValue = StringUnescaper.unquote(nextStateContext.literalString().getText()); - String variable = nextStateContext.ident().getText(); - Location location = toLocation(scope, node); - OperatorNode<SequenceOperator> next = OperatorNode.create(location, SequenceOperator.NEXT, continuationValue); - stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, next, variable)); - stmts.add(OperatorNode.create(location, StatementOperator.OUTPUT, variable)); - scope.defineVariable(location, variable); - break; + String variable = "result" + (++output); + boolean isCountVariable = false; + ParseTree outputStatement = node.getChild(0); + Location location = toLocation(scope, outputStatement); + for (int i = 1; i < outputStatement.getChildCount(); ++i) { + ParseTree child = outputStatement.getChild(i); + if ( getParseTreeIndex(child) != yqlplusParser.RULE_output_spec) + throw new ProgramCompileException( "Unknown statement attribute: " + child.toStringTree()); + + Output_specContext outputSpecContext = (Output_specContext) child; + variable = outputSpecContext.ident().getText(); + if (outputSpecContext.COUNT() != null) { + isCountVariable = true; } - case yqlplusParser.RULE_output_statement: - Source_statementContext source_statement = statementContext.output_statement().source_statement(); - OperatorNode<SequenceOperator> query; - if (source_statement.getChildCount() == 1) { - query = convertQuery( source_statement.query_statement().getChild(0), scope); - } else { - query = convertQuery(source_statement, scope); - } - String variable = "result" + (++output); - boolean isCountVariable = false; - OperatorNode<ExpressionOperator> pageSize = null; - ParseTree outputStatement = node.getChild(0); - Location location = toLocation(scope, outputStatement); - for (int i = 1; i < outputStatement.getChildCount(); ++i) { - ParseTree child = outputStatement.getChild(i); - switch (getParseTreeIndex(child)) { - case yqlplusParser.RULE_paged_clause: - Paged_clauseContext pagedContext = (Paged_clauseContext) child; - pageSize = convertExpr(pagedContext.fixed_or_parameter(), scope); - break; - case yqlplusParser.RULE_output_spec: - Output_specContext outputSpecContext = (Output_specContext) child; - variable = outputSpecContext.ident().getText(); - if (outputSpecContext.COUNT() != null) { - isCountVariable = true; - } - break; - default: - throw new ProgramCompileException( "Unknown statement attribute: " + child.toStringTree()); - } - } - scope.defineVariable(location, variable); - if (pageSize != null) { - query = OperatorNode.create(SequenceOperator.PAGE, query, pageSize); - } - stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable)); - stmts.add(OperatorNode.create(location, isCountVariable ? StatementOperator.COUNT:StatementOperator.OUTPUT, variable)); - } - break; - } - default: - throw new ProgramCompileException("Unknown program element: " + node.getText()); } + scope.defineVariable(location, variable); + stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable)); + stmts.add(OperatorNode.create(location, isCountVariable ? StatementOperator.COUNT:StatementOperator.OUTPUT, variable)); } - // traverse the tree, find all of the namespaced calls not covered by - // imports so we can - // define "implicit" import statements for them (to make engine - // implementation easier) return OperatorNode.create(StatementOperator.PROGRAM, stmts); } @@ -982,19 +611,19 @@ final class ProgramParser { private ProjectionBuilder readProjection(List<Field_defContext> fieldDefs, Scope scope) { if (null == fieldDefs) - throw new ProgramCompileException("Null fieldDefs"); + throw new ProgramCompileException("Null fieldDefs"); ProjectionBuilder proj = new ProjectionBuilder(); for (Field_defContext rulenode : fieldDefs) { // FIELD - // expression alias_def? - OperatorNode<ExpressionOperator> expr = convertExpr(rulenode.getChild(0), scope); + // expression alias_def? + OperatorNode<ExpressionOperator> expr = convertExpr(rulenode.getChild(0), scope); - String aliasName = null; - if (rulenode.getChildCount() > 1) { - // ^(ALIAS ID) - aliasName = rulenode.alias_def().ID().getText(); - } - proj.addField(aliasName, expr); + String aliasName = null; + if (rulenode.getChildCount() > 1) { + // ^(ALIAS ID) + aliasName = rulenode.alias_def().ID().getText(); + } + proj.addField(aliasName, expr); // no grammar for the other rule types at this time } return proj; @@ -1009,358 +638,345 @@ final class ProgramParser { } public OperatorNode<ExpressionOperator> convertExpr(ParseTree parseTree, Scope scope) { - switch (getParseTreeIndex(parseTree)) { - case yqlplusParser.RULE_vespa_grouping: { - ParseTree firstChild = parseTree.getChild(0); - if (getParseTreeIndex(firstChild) == yqlplusParser.RULE_annotation) { - ParseTree secondChild = parseTree.getChild(1); - OperatorNode<ExpressionOperator> annotation = convertExpr(((AnnotationContext) firstChild) - .constantMapExpression(), scope); - OperatorNode<ExpressionOperator> expr = OperatorNode.create(toLocation(scope, secondChild), - ExpressionOperator.VESPA_GROUPING, secondChild.getText()); - List<String> names = annotation.getArgument(0); - List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1); - for (int i = 0; i < names.size(); ++i) { - expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); - } - return expr; - } else { - return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VESPA_GROUPING, - firstChild.getText()); - } - } - case yqlplusParser.RULE_nullOperator: - return OperatorNode.create(ExpressionOperator.NULL); - case yqlplusParser.RULE_argument: - return convertExpr(parseTree.getChild(0), scope); - case yqlplusParser.RULE_fixed_or_parameter: { - ParseTree firstChild = parseTree.getChild(0); - if (getParseTreeIndex(firstChild) == yqlplusParser.INT) { - return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.LITERAL, Integer.valueOf(firstChild.getText())); - } else { - return convertExpr(firstChild, scope); - } - } - case yqlplusParser.RULE_constantMapExpression: { - List<ConstantPropertyNameAndValueContext> propertyList = ((ConstantMapExpressionContext) parseTree).constantPropertyNameAndValue(); - List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size()); - List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithExpectedSize(propertyList.size()); - for (ConstantPropertyNameAndValueContext child : propertyList) { - // : propertyName ':' expression[$expression::namespace] -> - // ^(PROPERTY propertyName expression) - names.add(StringUnescaper.unquote(child.getChild(0).getText())); - exprs.add(convertExpr(child.getChild(2), scope)); + switch (getParseTreeIndex(parseTree)) { + case yqlplusParser.RULE_vespa_grouping: { + ParseTree firstChild = parseTree.getChild(0); + if (getParseTreeIndex(firstChild) == yqlplusParser.RULE_annotation) { + ParseTree secondChild = parseTree.getChild(1); + OperatorNode<ExpressionOperator> annotation = convertExpr(((AnnotationContext) firstChild) + .constantMapExpression(), scope); + OperatorNode<ExpressionOperator> expr = OperatorNode.create(toLocation(scope, secondChild), + ExpressionOperator.VESPA_GROUPING, secondChild.getText()); + List<String> names = annotation.getArgument(0); + List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1); + for (int i = 0; i < names.size(); ++i) { + expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); + } + return expr; + } else { + return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VESPA_GROUPING, + firstChild.getText()); + } } - return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs); - } - case yqlplusParser.RULE_mapExpression: { - List<PropertyNameAndValueContext> propertyList = ((MapExpressionContext)parseTree).propertyNameAndValue(); - List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size()); - List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithCapacity(propertyList.size()); - for (PropertyNameAndValueContext child : propertyList) { - // : propertyName ':' expression[$expression::namespace] -> - // ^(PROPERTY propertyName expression) - names.add(StringUnescaper.unquote(child.getChild(0).getText())); - exprs.add(convertExpr(child.getChild(2), scope)); - } - return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs); - } - case yqlplusParser.RULE_constantArray: { - List<ConstantExpressionContext> expressionList = ((ConstantArrayContext)parseTree).constantExpression(); - List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size()); - for (ConstantExpressionContext expr : expressionList) { - values.add(convertExpr(expr, scope)); + case yqlplusParser.RULE_nullOperator: + return OperatorNode.create(ExpressionOperator.NULL); + case yqlplusParser.RULE_argument: + return convertExpr(parseTree.getChild(0), scope); + case yqlplusParser.RULE_fixed_or_parameter: { + ParseTree firstChild = parseTree.getChild(0); + if (getParseTreeIndex(firstChild) == yqlplusParser.INT) { + return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.LITERAL, Integer.valueOf(firstChild.getText())); + } else { + return convertExpr(firstChild, scope); + } + } + case yqlplusParser.RULE_constantMapExpression: { + List<ConstantPropertyNameAndValueContext> propertyList = ((ConstantMapExpressionContext) parseTree).constantPropertyNameAndValue(); + List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size()); + List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithExpectedSize(propertyList.size()); + for (ConstantPropertyNameAndValueContext child : propertyList) { + // : propertyName ':' expression[$expression::namespace] -> + // ^(PROPERTY propertyName expression) + names.add(StringUnescaper.unquote(child.getChild(0).getText())); + exprs.add(convertExpr(child.getChild(2), scope)); + } + return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs); + } + case yqlplusParser.RULE_mapExpression: { + List<PropertyNameAndValueContext> propertyList = ((MapExpressionContext)parseTree).propertyNameAndValue(); + List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size()); + List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithCapacity(propertyList.size()); + for (PropertyNameAndValueContext child : propertyList) { + // : propertyName ':' expression[$expression::namespace] -> + // ^(PROPERTY propertyName expression) + names.add(StringUnescaper.unquote(child.getChild(0).getText())); + exprs.add(convertExpr(child.getChild(2), scope)); + } + return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs); + } + case yqlplusParser.RULE_constantArray: { + List<ConstantExpressionContext> expressionList = ((ConstantArrayContext)parseTree).constantExpression(); + List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size()); + for (ConstantExpressionContext expr : expressionList) { + values.add(convertExpr(expr, scope)); + } + return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values); + } + case yqlplusParser.RULE_arrayLiteral: { + List<ExpressionContext> expressionList = ((ArrayLiteralContext) parseTree).expression(); + List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size()); + for (ExpressionContext expr : expressionList) { + values.add(convertExpr(expr, scope)); + } + return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values); + } + // dereferencedExpression: primaryExpression(indexref[in_select]| propertyref)* + case yqlplusParser.RULE_dereferencedExpression: { + DereferencedExpressionContext dereferencedExpression = (DereferencedExpressionContext) parseTree; + Iterator<ParseTree> it = dereferencedExpression.children.iterator(); + OperatorNode<ExpressionOperator> result = convertExpr(it.next(), scope); + while (it.hasNext()) { + ParseTree defTree = it.next(); + if (getParseTreeIndex(defTree) == yqlplusParser.RULE_propertyref) { + // DOT nm=ID + result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.PROPREF, result, defTree.getChild(1).getText()); + } else { + // indexref + result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.INDEX, result, convertExpr(defTree.getChild(1), scope)); + } + } + return result; + } + case yqlplusParser.RULE_primaryExpression: { + // ^(CALL namespaced_name arguments) + ParseTree firstChild = parseTree.getChild(0); + switch (getParseTreeIndex(firstChild)) { + case yqlplusParser.RULE_fieldref: { + return convertExpr(firstChild, scope); + } + case yqlplusParser.RULE_callExpresion: { + List<ArgumentContext> args = ((ArgumentsContext) firstChild.getChild(1)).argument(); + List<OperatorNode<ExpressionOperator>> arguments = Lists.newArrayListWithExpectedSize(args.size()); + for (ArgumentContext argContext : args) { + arguments.add(convertExpr(argContext.expression(),scope)); + } + return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.CALL, scope.resolvePath(readName((Namespaced_nameContext) firstChild.getChild(0))), arguments); + } + case yqlplusParser.RULE_parameter: + // external variable reference + return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VARREF, firstChild.getChild(1).getText()); + case yqlplusParser.RULE_scalar_literal: + case yqlplusParser.RULE_arrayLiteral: + case yqlplusParser.RULE_mapExpression: + return convertExpr(firstChild, scope); + case yqlplusParser.LPAREN: + return convertExpr(parseTree.getChild(1), scope); + } + break; + } + case yqlplusParser.RULE_parameter: { + // external variable reference + ParserRuleContext parameterContext = (ParserRuleContext) parseTree; + IdentContext identContext = parameterContext.getRuleContext(IdentContext.class, 0); + return OperatorNode.create(toLocation(scope, identContext), ExpressionOperator.VARREF, identContext.getText()); + } + case yqlplusParser.RULE_annotateExpression: { + //annotation logicalORExpression + AnnotationContext annotateExpressionContext = ((AnnotateExpressionContext)parseTree).annotation(); + OperatorNode<ExpressionOperator> annotation = convertExpr(annotateExpressionContext.constantMapExpression(), scope); + OperatorNode<ExpressionOperator> expr = convertExpr(parseTree.getChild(1), scope); + List<String> names = annotation.getArgument(0); + List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1); + for (int i = 0; i < names.size(); ++i) { + expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); + } + return expr; + } + case yqlplusParser.RULE_expression: { + return convertExpr(parseTree.getChild(0), scope); + } + case yqlplusParser.RULE_logicalANDExpression: + LogicalANDExpressionContext andExpressionContext = (LogicalANDExpressionContext) parseTree; + return readConjOp(ExpressionOperator.AND, andExpressionContext.equalityExpression(), scope); + case yqlplusParser.RULE_logicalORExpression: { + int childCount = parseTree.getChildCount(); + LogicalORExpressionContext logicalORExpressionContext = (LogicalORExpressionContext) parseTree; + if (childCount > 1) { + return readConjOrOp(ExpressionOperator.OR, logicalORExpressionContext, scope); + } else { + List<EqualityExpressionContext> equalityExpressionList = ((LogicalANDExpressionContext) parseTree.getChild(0)).equalityExpression(); + if (equalityExpressionList.size() > 1) { + return readConjOp(ExpressionOperator.AND, equalityExpressionList, scope); + } else { + return convertExpr(equalityExpressionList.get(0), scope); + } + } + } + case yqlplusParser.RULE_equalityExpression: { + EqualityExpressionContext equalityExpression = (EqualityExpressionContext) parseTree; + RelationalExpressionContext relationalExpressionContext = equalityExpression.relationalExpression(0); + OperatorNode<ExpressionOperator> expr = convertExpr(relationalExpressionContext, scope); + InNotInTargetContext inNotInTarget = equalityExpression.inNotInTarget(); + int childCount = equalityExpression.getChildCount(); + if (childCount == 1) { + return expr; + } + if (inNotInTarget != null) { + Literal_listContext literalListContext = inNotInTarget.literal_list(); + boolean isIN = equalityExpression.IN() != null; + if (literalListContext == null) { + Select_statementContext selectStatementContext = inNotInTarget.select_statement(); + OperatorNode<SequenceOperator> query = convertQuery(selectStatementContext, scope); + return OperatorNode.create(expr.getLocation(),isIN ? ExpressionOperator.IN_QUERY: ExpressionOperator.NOT_IN_QUERY, expr, query); + } else { + // we need to identify the type of the target; if it's a + // scalar we need to wrap it in a CREATE_ARRAY + // if it's already a CREATE ARRAY then it's fine, otherwise + // we need to know the variable type + // return readBinOp(node.getType() == yqlplusParser.IN ? + // ExpressionOperator.IN : ExpressionOperator.NOT_IN, node, + // scope); + return readBinOp(isIN ? ExpressionOperator.IN: ExpressionOperator.NOT_IN, equalityExpression.getChild(0), literalListContext, scope); + } + + } else { + ParseTree firstChild = equalityExpression.getChild(1); + if (equalityExpression.getChildCount() == 2) { + switch (getParseTreeIndex(firstChild)) { + case yqlplusParser.IS_NULL: + return readUnOp(ExpressionOperator.IS_NULL, relationalExpressionContext, scope); + case yqlplusParser.IS_NOT_NULL: + return readUnOp(ExpressionOperator.IS_NOT_NULL, relationalExpressionContext, scope); + } + } else { + switch (getParseTreeIndex(firstChild.getChild(0))) { + case yqlplusParser.EQ: + return readBinOp(ExpressionOperator.EQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.NEQ: + return readBinOp(ExpressionOperator.NEQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.LIKE: + return readBinOp(ExpressionOperator.LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.NOTLIKE: + return readBinOp(ExpressionOperator.NOT_LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.MATCHES: + return readBinOp(ExpressionOperator.MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.NOTMATCHES: + return readBinOp(ExpressionOperator.NOT_MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.CONTAINS: + return readBinOp(ExpressionOperator.CONTAINS, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + } + } + + } + break; + } + case yqlplusParser.RULE_relationalExpression: { + RelationalExpressionContext relationalExpressionContext = (RelationalExpressionContext) parseTree; + RelationalOpContext opContext = relationalExpressionContext.relationalOp(); + if (opContext != null) { + switch (getParseTreeIndex(relationalExpressionContext.relationalOp().getChild(0))) { + case yqlplusParser.LT: + return readBinOp(ExpressionOperator.LT, parseTree, scope); + case yqlplusParser.LTEQ: + return readBinOp(ExpressionOperator.LTEQ, parseTree, scope); + case yqlplusParser.GT: + return readBinOp(ExpressionOperator.GT, parseTree, scope); + case yqlplusParser.GTEQ: + return readBinOp(ExpressionOperator.GTEQ, parseTree, scope); + } + } else { + return convertExpr(relationalExpressionContext.additiveExpression(0), scope); + } + } + break; + case yqlplusParser.RULE_additiveExpression: + case yqlplusParser.RULE_multiplicativeExpression: { + if (parseTree.getChildCount() > 1) { + String opStr = parseTree.getChild(1).getText(); + switch (opStr) { + case "+": + return readBinOp(ExpressionOperator.ADD, parseTree, scope); + case "-": + return readBinOp(ExpressionOperator.SUB, parseTree, scope); + case "/": + return readBinOp(ExpressionOperator.DIV, parseTree, scope); + case "*": + return readBinOp(ExpressionOperator.MULT, parseTree, scope); + case "%": + return readBinOp(ExpressionOperator.MOD, parseTree, scope); + default: + if (parseTree.getChild(0) instanceof UnaryExpressionContext) { + return convertExpr(parseTree.getChild(0), scope); + } else { + throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.toStringTree()); + } + } + } else { + if (parseTree.getChild(0) instanceof UnaryExpressionContext) { + return convertExpr(parseTree.getChild(0), scope); + } else if (parseTree.getChild(0) instanceof MultiplicativeExpressionContext) { + return convertExpr(parseTree.getChild(0), scope); + } else { + throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.getText()); + } + } + } + case yqlplusParser.RULE_unaryExpression: { + if (1 == parseTree.getChildCount()) { + return convertExpr(parseTree.getChild(0), scope); + } else if (2 == parseTree.getChildCount()) { + if ("-".equals(parseTree.getChild(0).getText())) { + return readUnOp(ExpressionOperator.NEGATE, parseTree, scope); + } else if ("!".equals(parseTree.getChild(0).getText())) { + return readUnOp(ExpressionOperator.NOT, parseTree, scope); + } + throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown unary operator " + parseTree.getText()); + } else { + throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown child count " + parseTree.getChildCount() + " of " + parseTree.getText()); + } + } + case yqlplusParser.RULE_fieldref: { + // all in-scope data sources should be defined in scope + // the 'first' field in a namespaced reference must be: + // - a field name if (and only if) there is exactly one data source + // in scope OR + // - an alias name, which will be followed by a field name + // ^(FIELDREF<FieldReference>[$expression::namespace] + // namespaced_name) + List<String> path = readName((Namespaced_nameContext) parseTree.getChild(0)); + Location loc = toLocation(scope, parseTree.getChild(0)); + String alias = path.get(0); + OperatorNode<ExpressionOperator> result = null; + int start = 0; + if (scope.isCursor(alias)) { + if (path.size() > 1) { + result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(1)); + start = 2; + } else { + result = OperatorNode.create(loc, ExpressionOperator.READ_RECORD, alias); + start = 1; + } + } else if (scope.isBound(alias)) { + return OperatorNode.create(loc, ExpressionOperator.READ_MODULE, scope.getBinding(alias).toPathWith(path.subList(1, path.size()))); + } else if (scope.getCursors().size() == 1) { + alias = scope.getCursors().iterator().next(); + result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(0)); + start = 1; + } else { + // ah ha, we can't end up with a 'loose' UDF call because it + // won't be a module or known alias + // so we need not support implicit imports for constants used in + // UDFs + throw new ProgramCompileException(loc, "Unknown field or alias '%s'", alias); + } + for (int idx = start; idx < path.size(); ++idx) { + result = OperatorNode.create(loc, ExpressionOperator.PROPREF, result, path.get(idx)); + } + return result; } - return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values); + case yqlplusParser.RULE_scalar_literal: + return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.LITERAL, convertLiteral((Scalar_literalContext) parseTree)); + case yqlplusParser.RULE_constantExpression: + return convertExpr(parseTree.getChild(0), scope); + case yqlplusParser.RULE_literal_list: + if (getParseTreeIndex(parseTree.getChild(1)) == yqlplusParser.RULE_array_parameter) { + return convertExpr(parseTree.getChild(1), scope); + } else { + List<Literal_elementContext> elements = ((Literal_listContext) parseTree).literal_element(); + ParseTree firldElement = elements.get(0).getChild(0); + if (elements.size() == 1 && scope.getParser().isArrayParameter(firldElement)) { + return convertExpr(firldElement, scope); + } else { + List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(elements.size()); + for (Literal_elementContext child : elements) { + values.add(convertExpr(child.getChild(0), scope)); + } + return OperatorNode.create(toLocation(scope, elements.get(0)),ExpressionOperator.ARRAY, values); + } + } } - case yqlplusParser.RULE_arrayLiteral: { - List<ExpressionContext> expressionList = ((ArrayLiteralContext) parseTree).expression(); - List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size()); - for (ExpressionContext expr : expressionList) { - values.add(convertExpr(expr, scope)); - } - return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values); - } - //dereferencedExpression: primaryExpression(indexref[in_select]| propertyref)* - case yqlplusParser.RULE_dereferencedExpression: { - DereferencedExpressionContext dereferencedExpression = (DereferencedExpressionContext) parseTree; - Iterator<ParseTree> it = dereferencedExpression.children.iterator(); - OperatorNode<ExpressionOperator> result = convertExpr(it.next(), scope); - while (it.hasNext()) { - ParseTree defTree = it.next(); - if (getParseTreeIndex(defTree) == yqlplusParser.RULE_propertyref) { - //DOT nm=ID - result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.PROPREF, result, defTree.getChild(1).getText()); - } else { - //indexref - result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.INDEX, result, convertExpr(defTree.getChild(1), scope)); - } - } - return result; - } - case yqlplusParser.RULE_primaryExpression: { - // ^(CALL namespaced_name arguments) - ParseTree firstChild = parseTree.getChild(0); - switch (getParseTreeIndex(firstChild)) { - case yqlplusParser.RULE_fieldref: { - return convertExpr(firstChild, scope); - } - case yqlplusParser.RULE_callExpresion: { - List<ArgumentContext> args = ((ArgumentsContext) firstChild.getChild(1)).argument(); - List<OperatorNode<ExpressionOperator>> arguments = Lists.newArrayListWithExpectedSize(args.size()); - for (ArgumentContext argContext : args) { - arguments.add(convertExpr(argContext.expression(),scope)); - } - return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.CALL, scope.resolvePath(readName((Namespaced_nameContext) firstChild.getChild(0))), arguments); - } - // TODO add processing this is not implemented in V3 - // case yqlplusParser.APPLY: - - case yqlplusParser.RULE_parameter: - // external variable reference - return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VARREF, firstChild.getChild(1).getText()); - case yqlplusParser.RULE_scalar_literal: - case yqlplusParser.RULE_arrayLiteral: - case yqlplusParser.RULE_mapExpression: - return convertExpr(firstChild, scope); - case yqlplusParser.LPAREN: - return convertExpr(parseTree.getChild(1), scope); - } - break; - } - - // TODO: Temporarily disable CAST - think through how types are named - // case yqlplusParser.CAST: { - // - // return new Cast() - // } - // return new CastExpression(payload); - case yqlplusParser.RULE_parameter: { - // external variable reference - ParserRuleContext parameterContext = (ParserRuleContext) parseTree; - IdentContext identContext = parameterContext.getRuleContext(IdentContext.class, 0); - return OperatorNode.create(toLocation(scope, identContext), ExpressionOperator.VARREF, identContext.getText()); - } - case yqlplusParser.RULE_annotateExpression: { - //annotation logicalORExpression - AnnotationContext annotateExpressionContext = ((AnnotateExpressionContext)parseTree).annotation(); - OperatorNode<ExpressionOperator> annotation = convertExpr(annotateExpressionContext.constantMapExpression(), scope); - OperatorNode<ExpressionOperator> expr = convertExpr(parseTree.getChild(1), scope); - List<String> names = annotation.getArgument(0); - List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1); - for (int i = 0; i < names.size(); ++i) { - expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); - } - return expr; - } - case yqlplusParser.RULE_expression: { - return convertExpr(parseTree.getChild(0), scope); - } - case yqlplusParser.RULE_logicalANDExpression: - LogicalANDExpressionContext andExpressionContext = (LogicalANDExpressionContext) parseTree; - return readConjOp(ExpressionOperator.AND, andExpressionContext.equalityExpression(), scope); - case yqlplusParser.RULE_logicalORExpression: { - int childCount = parseTree.getChildCount(); - LogicalORExpressionContext logicalORExpressionContext = (LogicalORExpressionContext) parseTree; - if (childCount > 1) { - return readConjOrOp(ExpressionOperator.OR, logicalORExpressionContext, scope); - } else { - List<EqualityExpressionContext> equalityExpressionList = ((LogicalANDExpressionContext) parseTree.getChild(0)).equalityExpression(); - if (equalityExpressionList.size() > 1) { - return readConjOp(ExpressionOperator.AND, equalityExpressionList, scope); - } else { - return convertExpr(equalityExpressionList.get(0), scope); - } - } - } - case yqlplusParser.RULE_equalityExpression: { - EqualityExpressionContext equalityExpression = (EqualityExpressionContext) parseTree; - RelationalExpressionContext relationalExpressionContext = equalityExpression.relationalExpression(0); - OperatorNode<ExpressionOperator> expr = convertExpr(relationalExpressionContext, scope); - InNotInTargetContext inNotInTarget = equalityExpression.inNotInTarget(); - int childCount = equalityExpression.getChildCount(); - if (childCount == 1) { - return expr; - } - if (inNotInTarget != null) { - Literal_listContext literalListContext = inNotInTarget.literal_list(); - boolean isIN = equalityExpression.IN() != null; - if (literalListContext == null) { - Select_statementContext selectStatementContext = inNotInTarget.select_statement(); - OperatorNode<SequenceOperator> query = convertQuery(selectStatementContext, scope); - return OperatorNode.create(expr.getLocation(),isIN ? ExpressionOperator.IN_QUERY: ExpressionOperator.NOT_IN_QUERY, expr, query); - } else { - // we need to identify the type of the target; if it's a - // scalar we need to wrap it in a CREATE_ARRAY - // if it's already a CREATE ARRAY then it's fine, otherwise - // we need to know the variable type - // return readBinOp(node.getType() == yqlplusParser.IN ? - // ExpressionOperator.IN : ExpressionOperator.NOT_IN, node, - // scope); - return readBinOp(isIN ? ExpressionOperator.IN: ExpressionOperator.NOT_IN, equalityExpression.getChild(0), literalListContext, scope); - } - - } else { - ParseTree firstChild = equalityExpression.getChild(1); - if (equalityExpression.getChildCount() == 2) { - switch (getParseTreeIndex(firstChild)) { - case yqlplusParser.IS_NULL: - return readUnOp(ExpressionOperator.IS_NULL, relationalExpressionContext, scope); - case yqlplusParser.IS_NOT_NULL: - return readUnOp(ExpressionOperator.IS_NOT_NULL, relationalExpressionContext, scope); - } - } else { - switch (getParseTreeIndex(firstChild.getChild(0))) { - case yqlplusParser.EQ: - return readBinOp(ExpressionOperator.EQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.NEQ: - return readBinOp(ExpressionOperator.NEQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.LIKE: - return readBinOp(ExpressionOperator.LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.NOTLIKE: - return readBinOp(ExpressionOperator.NOT_LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.MATCHES: - return readBinOp(ExpressionOperator.MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.NOTMATCHES: - return readBinOp(ExpressionOperator.NOT_MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.CONTAINS: - return readBinOp(ExpressionOperator.CONTAINS, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - } - } - - } - break; - } - case yqlplusParser.RULE_relationalExpression: { - RelationalExpressionContext relationalExpressionContext = (RelationalExpressionContext) parseTree; - RelationalOpContext opContext = relationalExpressionContext.relationalOp(); - if (opContext != null) { - switch (getParseTreeIndex(relationalExpressionContext.relationalOp().getChild(0))) { - case yqlplusParser.LT: - return readBinOp(ExpressionOperator.LT, parseTree, scope); - case yqlplusParser.LTEQ: - return readBinOp(ExpressionOperator.LTEQ, parseTree, scope); - case yqlplusParser.GT: - return readBinOp(ExpressionOperator.GT, parseTree, scope); - case yqlplusParser.GTEQ: - return readBinOp(ExpressionOperator.GTEQ, parseTree, scope); - } - } else { - return convertExpr(relationalExpressionContext.additiveExpression(0), scope); - } - } - break; - case yqlplusParser.RULE_additiveExpression: - case yqlplusParser.RULE_multiplicativeExpression: { - if (parseTree.getChildCount() > 1) { - String opStr = parseTree.getChild(1).getText(); - switch (opStr) { - case "+": - return readBinOp(ExpressionOperator.ADD, parseTree, scope); - case "-": - return readBinOp(ExpressionOperator.SUB, parseTree, scope); - case "/": - return readBinOp(ExpressionOperator.DIV, parseTree, scope); - case "*": - return readBinOp(ExpressionOperator.MULT, parseTree, scope); - case "%": - return readBinOp(ExpressionOperator.MOD, parseTree, scope); - default: - if (parseTree.getChild(0) instanceof UnaryExpressionContext) { - return convertExpr(parseTree.getChild(0), scope); - } else { - throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.toStringTree()); - } - } - } else { - if (parseTree.getChild(0) instanceof UnaryExpressionContext) { - return convertExpr(parseTree.getChild(0), scope); - } else if (parseTree.getChild(0) instanceof MultiplicativeExpressionContext) { - return convertExpr(parseTree.getChild(0), scope); - } else { - throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.getText()); - } - } - } - case yqlplusParser.RULE_unaryExpression: { - if (1 == parseTree.getChildCount()) { - return convertExpr(parseTree.getChild(0), scope); - } else if (2 == parseTree.getChildCount()) { - if ("-".equals(parseTree.getChild(0).getText())) { - return readUnOp(ExpressionOperator.NEGATE, parseTree, scope); - } else if ("!".equals(parseTree.getChild(0).getText())) { - return readUnOp(ExpressionOperator.NOT, parseTree, scope); - } - throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown unary operator " + parseTree.getText()); - } else { - throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown child count " + parseTree.getChildCount() + " of " + parseTree.getText()); - } - } - case yqlplusParser.RULE_fieldref: - case yqlplusParser.RULE_joinDereferencedExpression: { - // all in-scope data sources should be defined in scope - // the 'first' field in a namespaced reference must be: - // - a field name if (and only if) there is exactly one data source - // in scope OR - // - an alias name, which will be followed by a field name - // ^(FIELDREF<FieldReference>[$expression::namespace] - // namespaced_name) - List<String> path = readName((Namespaced_nameContext) parseTree.getChild(0)); - Location loc = toLocation(scope, parseTree.getChild(0)); - String alias = path.get(0); - OperatorNode<ExpressionOperator> result = null; - int start = 0; - if (scope.isCursor(alias)) { - if (path.size() > 1) { - result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(1)); - start = 2; - } else { - result = OperatorNode.create(loc, ExpressionOperator.READ_RECORD, alias); - start = 1; - } - } else if (scope.isBound(alias)) { - return OperatorNode.create(loc, ExpressionOperator.READ_MODULE, scope.getBinding(alias).toPathWith(path.subList(1, path.size()))); - } else if (scope.getCursors().size() == 1) { - alias = scope.getCursors().iterator().next(); - result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(0)); - start = 1; - } else { - // ah ha, we can't end up with a 'loose' UDF call because it - // won't be a module or known alias - // so we need not support implicit imports for constants used in - // UDFs - throw new ProgramCompileException(loc, "Unknown field or alias '%s'", alias); - } - for (int idx = start; idx < path.size(); ++idx) { - result = OperatorNode.create(loc, ExpressionOperator.PROPREF, result, path.get(idx)); - } - return result; - } - case yqlplusParser.RULE_scalar_literal: - return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.LITERAL, convertLiteral((Scalar_literalContext) parseTree)); - case yqlplusParser.RULE_insert_values: - return readValues((Insert_valuesContext) parseTree, scope); - case yqlplusParser.RULE_constantExpression: - return convertExpr(parseTree.getChild(0), scope); - case yqlplusParser.RULE_literal_list: - if (getParseTreeIndex(parseTree.getChild(1)) == yqlplusParser.RULE_array_parameter) { - return convertExpr(parseTree.getChild(1), scope); - } else { - List<Literal_elementContext> elements = ((Literal_listContext) parseTree).literal_element(); - ParseTree firldElement = elements.get(0).getChild(0); - if (elements.size() == 1 && scope.getParser().isArrayParameter(firldElement)) { - return convertExpr(firldElement, scope); - } else { - List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(elements.size()); - for (Literal_elementContext child : elements) { - values.add(convertExpr(child.getChild(0), scope)); - } - return OperatorNode.create(toLocation(scope, elements.get(0)),ExpressionOperator.ARRAY, values); - } - } - } - throw new ProgramCompileException(toLocation(scope, parseTree), - "Unknown expression type: " + parseTree.getText()); + throw new ProgramCompileException(toLocation(scope, parseTree), + "Unknown expression type: " + parseTree.getText()); } public Object convertLiteral(Scalar_literalContext literal) { @@ -1462,77 +1078,7 @@ final class ProgramParser { } } - private OperatorNode<ExpressionOperator> readValues(Field_names_specContext nameDefs, Field_values_specContext values, Scope scope) { - List<Field_defContext> fieldDefs = nameDefs.field_def(); - List<ExpressionContext> valueDefs = values.expression(); - assert fieldDefs.size() == valueDefs.size(); - List<String> fieldNames; - List<OperatorNode<ExpressionOperator>> fieldValues; - int numPairs = fieldDefs.size(); - fieldNames = Lists.newArrayListWithExpectedSize(numPairs); - fieldValues = Lists.newArrayListWithExpectedSize(numPairs); - for (int i = 0; i < numPairs; i++) { - fieldNames.add((String) convertExpr(fieldDefs.get(i).expression(), scope).getArgument(1)); - fieldValues.add(convertExpr(valueDefs.get(i), scope)); - } - return OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues); - } - - private OperatorNode<ExpressionOperator> readValues(ParserRuleContext node, Scope scope) { - List<String> fieldNames; - List<OperatorNode<ExpressionOperator>> fieldValues; - if (node.getRuleIndex() == yqlplusParser.RULE_field_def) { - Field_defContext fieldDefContext = (Field_defContext)node; - //TODO double check - fieldNames = Lists.newArrayListWithExpectedSize(node.getChildCount()); - fieldValues = Lists.newArrayListWithExpectedSize(node.getChildCount()); - for (int i = 0; i < node.getChildCount(); i++) { - fieldNames.add((String) convertExpr(node.getChild(i).getChild(0).getChild(0), scope).getArgument(1)); - fieldValues.add(convertExpr(node.getChild(i).getChild(0).getChild(1), scope)); - } - } else { - assert node.getChildCount() % 2 == 0; - int numPairs = node.getChildCount() / 2; - fieldNames = Lists.newArrayListWithExpectedSize(numPairs); - fieldValues = Lists.newArrayListWithExpectedSize(numPairs); - for (int i = 0; i < numPairs; i++) { - fieldNames.add((String) convertExpr(node.getChild(i).getChild(0), scope).getArgument(1)); - fieldValues.add(convertExpr(node.getChild(numPairs + i), scope)); - } - } - return OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues); - } - - /* - * Converts node list - * - * a_name, b_name, c_name, a_value_1, b_value_1, c_value_1, a_value_2, b_value_2, c_value2, a_value_3, b_value_3, c_value_3 - * - * into corresponding constant sequence: - * - * [ { a_name : a_value_1, b_name : b_value_1, c_name : c_value_1 }, ... ] - * - */ - private OperatorNode<SequenceOperator> readBatchValues(Field_names_specContext nameDefs, List<Field_values_group_specContext> valueGroups, Scope scope) { - List<Field_defContext> nameContexts = nameDefs.field_def(); - List<String> fieldNames = Lists.newArrayList(); - for (Field_defContext nameContext:nameContexts) { - fieldNames.add((String) convertExpr(nameContext.getChild(0), scope).getArgument(1)); - } - List<OperatorNode> records = Lists.newArrayList(); - for (Field_values_group_specContext valueGorup:valueGroups) { - List<ExpressionContext> expressionList = valueGorup.expression(); - List<OperatorNode<ExpressionOperator>> fieldValues = Lists.newArrayListWithExpectedSize(expressionList.size()); - for (ExpressionContext expressionContext:expressionList) { - fieldValues.add(convertExpr(expressionContext, scope)); - } - records.add(OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues)); - } - // Return constant sequence of records with the given name/values - return OperatorNode.create(SequenceOperator.EVALUATE, OperatorNode.create(ExpressionOperator.ARRAY, records)); - } - - /* + /** * Scans the given node for READ_FIELD expressions. * * TODO: Search recursively and consider additional operators @@ -1557,4 +1103,5 @@ final class ProgramParser { } return readFieldList; } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java index 204ccff7fb3..02175425808 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java @@ -192,7 +192,7 @@ public class ItemsCommonStuffTestCase { assertEquals("SAND", i.getName()); i = new WeakAndItem(); assertEquals(ItemType.WEAK_AND, i.getItemType()); - assertEquals("WAND", i.getName()); + assertEquals("WEAKAND", i.getName()); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java index f221cbb63b1..6afea895f3a 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -2480,17 +2480,17 @@ public class ParseTestCase { @Test public void testSimpleWandAdvanced() { - tester.assertParsed("WAND(100) foo bar baz", "foo wand bar wand baz", Query.Type.ADVANCED); + tester.assertParsed("WEAKAND(100) foo bar baz", "foo wand bar wand baz", Query.Type.ADVANCED); } @Test public void testSimpleWandAdvancedWithNonDefaultN() { - tester.assertParsed("WAND(32) foo bar baz", "foo wand(32) bar wand(32) baz", Query.Type.ADVANCED); + tester.assertParsed("WEAKAND(32) foo bar baz", "foo weakand(32) bar weakand(32) baz", Query.Type.ADVANCED); } @Test public void testSimpleWandAdvancedWithNonDefaultNAndWeights() { - tester.assertParsed("WAND(32) foo!32 bar!64 baz", "foo!32 wand(32) bar!64 wand(32) baz", Query.Type.ADVANCED); + tester.assertParsed("WEAKAND(32) foo!32 bar!64 baz", "foo!32 weakand(32) bar!64 weakand(32) baz", Query.Type.ADVANCED); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java index d3b64100c1e..e7a2a4f3ef8 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java @@ -29,7 +29,7 @@ public class QueryCanonicalizerTestCase { CompositeItem root = new WeakAndItem(); root.addItem(new WordItem("word")); - assertCanonicalized("WAND(100) word", null, root); + assertCanonicalized("WEAKAND(100) word", null, root); } @Test @@ -108,6 +108,47 @@ public class QueryCanonicalizerTestCase { } @Test + public void testMultilevelWeakAndCollapsing() { + CompositeItem root = new WeakAndItem(); + CompositeItem l1 = new WeakAndItem(); + CompositeItem l2 = new WeakAndItem(); + CompositeItem l3 = new WeakAndItem(); + CompositeItem l4 = new WeakAndItem(); + + root.addItem(l1); + + l1.addItem(l2); + l1.addItem(new WordItem("l1")); + + l2.addItem(l3); + l2.addItem(new WordItem("l2")); + + l3.addItem(l4); + l3.addItem(new WordItem("l3")); + + l4.addItem(new WordItem("l4")); + + assertCanonicalized("WEAKAND(100) l4 l3 l2 l1", null, root); + } + + @Test + public void testWeakAndCollapsingRequireSameNAndIndex() { + CompositeItem root = new WeakAndItem(10); + CompositeItem l1 = new WeakAndItem(100); + CompositeItem l2 = new WeakAndItem(100); + l2.setIndexName("other"); + + root.addItem(l1); + + l1.addItem(l2); + l1.addItem(new WordItem("l1")); + + l2.addItem(new WordItem("l2")); + + assertCanonicalized("WEAKAND(10) (WEAKAND(100) (WEAKAND(100) l2) l1)", null, root); + } + + @Test public void testNullRoot() { assertCanonicalized(null, "No query", new Query()); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index eaafb2d8b8a..943390cb10c 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -6,6 +6,7 @@ import com.yahoo.prelude.fastsearch.test.MockMetric; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.cluster.ClusterMonitor; +import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.PingFactory; import com.yahoo.search.dispatch.searchcluster.Pinger; @@ -166,7 +167,7 @@ public class DispatcherTest { boolean nonEmpty = events[step].returnInvoker(nodes, acceptIncompleteCoverage); step++; if (nonEmpty) { - return Optional.of(new MockInvoker(nodes.get(0).key())); + return Optional.of(new MockInvoker(nodes.get(0).key(), groupId)); } else { return Optional.empty(); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index 8cab7884152..730aa0800e7 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -44,10 +44,11 @@ import static org.junit.Assert.fail; * @author ollivir */ public class InterleavedSearchInvokerTest { - private ManualClock clock = new ManualClock(Instant.now()); - private Query query = new TestQuery(); - private LinkedList<Event> expectedEvents = new LinkedList<>(); - private List<SearchInvoker> invokers = new ArrayList<>(); + + private final ManualClock clock = new ManualClock(Instant.now()); + private final Query query = new TestQuery(); + private final LinkedList<Event> expectedEvents = new LinkedList<>(); + private final List<SearchInvoker> invokers = new ArrayList<>(); @Test public void requireThatAdaptiveTimeoutsAreNotUsedWithFullCoverageRequirement() throws IOException { diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 36b476e2936..d0ba396ef94 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -26,6 +26,7 @@ import static org.junit.Assert.assertThat; * @author ollivir */ public class LoadBalancerTest { + @Test public void requireThatLoadBalancerServesSingleNodeSetups() { Node n1 = new Node(0, "test-node1", 0); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java index 459dcc83ab0..53d1a2457d0 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java @@ -12,20 +12,32 @@ import com.yahoo.search.searchchain.Execution; import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.OptionalInt; class MockInvoker extends SearchInvoker { + private final Coverage coverage; + private final OptionalInt groupId; private Query query; private List<Hit> hits; int hitsRequested; - protected MockInvoker(int key, Coverage coverage) { + protected MockInvoker(int key, Coverage coverage, OptionalInt groupId) { super(Optional.of(new Node(key, "?", 0))); this.coverage = coverage; + this.groupId = groupId; + } + + protected MockInvoker(int key, OptionalInt groupId) { + this(key, null, groupId); + } + + protected MockInvoker(int key, Coverage coverage) { + this(key, coverage, OptionalInt.empty()); } protected MockInvoker(int key) { - this(key, null); + this(key, null, OptionalInt.empty()); } MockInvoker setHits(List<Hit> hits) { @@ -33,6 +45,9 @@ class MockInvoker extends SearchInvoker { return this; } + /** Returns the group to be invoked, if known */ + public OptionalInt groupId() { return groupId; } + @Override protected Object sendSearchRequest(Query query, Object context) throws IOException { this.query = query; @@ -62,4 +77,11 @@ class MockInvoker extends SearchInvoker { @Override protected void release() { } + + @Override + public String toString() { + return "invoker with key " + distributionKey() + + (groupId().isPresent() ? " of group " + groupId().getAsInt() : ""); + } + }
\ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index fe43658ee75..9d96b2302d7 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -114,6 +114,7 @@ public class MockSearchCluster extends SearchCluster { public static DispatchConfig createDispatchConfig(double minSearchCoverage, Node... nodes) { return createDispatchConfig(minSearchCoverage, Arrays.asList(nodes)); } + public static DispatchConfig createDispatchConfig(double minSearchCoverage, List<Node> nodes) { DispatchConfig.Builder builder = new DispatchConfig.Builder(); builder.minActivedocsPercentage(88.0); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java new file mode 100644 index 00000000000..6338107d4b6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java @@ -0,0 +1,89 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch.searchcluster; + +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class SearchClusterCoverageTest { + + @Test + public void two_groups_equal_docs() { + var tester = new SearchClusterTester(2, 3); + + tester.setDocsPerNode(100, 0); + tester.setDocsPerNode(100, 1); + tester.pingIterationCompleted(); + assertTrue(tester.group(0).hasSufficientCoverage()); + assertTrue(tester.group(1).hasSufficientCoverage()); + } + + @Test + public void two_groups_one_missing_docs() { + var tester = new SearchClusterTester(2, 3); + + tester.setDocsPerNode(100, 0); + tester.setDocsPerNode( 70, 1); + tester.pingIterationCompleted(); + assertTrue(tester.group(0).hasSufficientCoverage()); + assertFalse(tester.group(1).hasSufficientCoverage()); + } + + @Test + public void three_groups_one_missing_docs() { + var tester = new SearchClusterTester(3, 3); + + tester.setDocsPerNode(100, 0); + tester.setDocsPerNode( 87, 1); // min is set to 88 in MockSearchCluster + tester.setDocsPerNode(100, 2); + tester.pingIterationCompleted(); + assertTrue(tester.group(0).hasSufficientCoverage()); + assertFalse(tester.group(1).hasSufficientCoverage()); + assertTrue(tester.group(2).hasSufficientCoverage()); + } + + @Test + public void three_groups_one_missing_docs_but_too_few() { + var tester = new SearchClusterTester(3, 3); + + tester.setDocsPerNode(100, 0); + tester.setDocsPerNode( 89, 1); // min is set to 88 in MockSearchCluster + tester.setDocsPerNode(100, 2); + tester.pingIterationCompleted(); + assertTrue(tester.group(0).hasSufficientCoverage()); + assertTrue(tester.group(1).hasSufficientCoverage()); + assertTrue(tester.group(2).hasSufficientCoverage()); + } + + @Test + public void three_groups_one_has_too_many_docs() { + var tester = new SearchClusterTester(3, 3); + + tester.setDocsPerNode(100, 0); + tester.setDocsPerNode(150, 1); + tester.setDocsPerNode(100, 2); + tester.pingIterationCompleted(); + assertTrue(tester.group(0).hasSufficientCoverage()); + assertTrue(tester.group(1).hasSufficientCoverage()); + assertTrue(tester.group(2).hasSufficientCoverage()); + } + + @Test + public void three_groups_one_has_a_node_down() { + var tester = new SearchClusterTester(3, 3); + + tester.setDocsPerNode(100, 0); + tester.setDocsPerNode(150, 1); + tester.setDocsPerNode(100, 2); + tester.setWorking(1, 1, false); + tester.pingIterationCompleted(); + assertTrue(tester.group(0).hasSufficientCoverage()); + assertFalse(tester.group(1).hasSufficientCoverage()); + assertTrue(tester.group(2).hasSufficientCoverage()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index c6fd48836fe..48134094faf 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -28,7 +28,7 @@ import static org.junit.Assert.assertTrue; */ public class SearchClusterTest { - static class State implements AutoCloseable{ + static class State implements AutoCloseable { final String clusterId; final int nodesPerGroup; diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTester.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTester.java new file mode 100644 index 00000000000..5e7ecb854ff --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTester.java @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch.searchcluster; + +import com.yahoo.search.dispatch.MockSearchCluster; + +public class SearchClusterTester { + + private final SearchCluster cluster; + + public SearchClusterTester(int groups, int nodesPerGroup) { + cluster = new MockSearchCluster("1", groups, nodesPerGroup); + } + + public void pingIterationCompleted() { + cluster.pingIterationCompleted(); + } + + public Group group(int id) { + return cluster.group(id).get(); + } + + public void setWorking(int group, int node, boolean working) { + cluster.group(group).get().nodes().get(node).setWorking(working); + } + + public void setDocsPerNode(int docs, int groupId) { + for (Node node : cluster.groups().get(groupId).nodes()) { + node.setWorking(true); + node.setActiveDocuments(docs); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 24c8e040051..dd2d27eb66c 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -631,14 +631,14 @@ public class YqlParserTestCase { @SuppressWarnings("deprecation") public void testWeakAnd() { assertParse("select foo from bar where weakAnd(a contains \"A\", b contains \"B\");", - "WAND(100) a:A b:B"); + "WEAKAND(100) a:A b:B"); assertParse("select foo from bar where [{\"targetHits\": 37}]weakAnd(a contains \"A\", " + "b contains \"B\");", - "WAND(37) a:A b:B"); + "WEAKAND(37) a:A b:B"); QueryTree tree = parse("select foo from bar where [{\"scoreThreshold\": 41}]weakAnd(a " + "contains \"A\", b contains \"B\");"); - assertEquals("WAND(100) a:A b:B", tree.toString()); + assertEquals("WEAKAND(100) a:A b:B", tree.toString()); assertEquals(WeakAndItem.class, tree.getRoot().getClass()); assertEquals(41, ((WeakAndItem)tree.getRoot()).getScoreThreshold()); } diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 7549c67d0ae..d1e46a6a8c2 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -555,12 +555,12 @@ public class SelectTestCase { @SuppressWarnings("deprecation") public void testWeakAnd() { assertParse("{ \"weakAnd\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }", - "WAND(100) a:A b:B"); + "WEAKAND(100) a:A b:B"); assertParse("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"targetHits\": 37} }}", - "WAND(37) a:A b:B"); + "WEAKAND(37) a:A b:B"); QueryTree tree = parseWhere("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"scoreThreshold\": 41}}}"); - assertEquals("WAND(100) a:A b:B", tree.toString()); + assertEquals("WEAKAND(100) a:A b:B", tree.toString()); assertEquals(WeakAndItem.class, tree.getRoot().getClass()); assertEquals(41, ((WeakAndItem)tree.getRoot()).getScoreThreshold()); } |