Publish

3 files changed, 2002 insertions, 0 deletions

diff --git a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj
new file mode 100644
index 00000000000..82c95578147
--- /dev/null
+++ b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj
@@ -0,0 +1,649 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author bratseth
+ */
+options {
+    CACHE_TOKENS    = true;
+    DEBUG_PARSER    = false;
+    ERROR_REPORTING = true;
+    STATIC          = false;
+    UNICODE_INPUT   = true;
+}
+
+PARSER_BEGIN(SemanticsParser)
+
+package com.yahoo.prelude.semantics.parser;
+
+import com.yahoo.javacc.UnicodeUtilities;
+import com.yahoo.prelude.semantics.*;
+import com.yahoo.prelude.semantics.rule.*;
+import com.yahoo.prelude.query.TermType;
+
+public class SemanticsParser {
+
+}
+
+PARSER_END(SemanticsParser)
+
+SKIP :
+{
+    " " | "\f" | "\r" | "\t"
+}
+
+SPECIAL_TOKEN :
+{
+    <SINGLE_LINE_COMMENT: "#" (~["\n","\r"])* ("\n"|"\r"|"\r\n")? >
+}
+
+TOKEN :
+{
+    <NUMBER: <DECIMAL> (["l","L"])? | <HEX> (["l","L"])? | <OCTAL> (["l","L"])?> |
+        <#DECIMAL: ["1"-"9"] (["0"-"9"])*> |
+        <#HEX: "0" ["x","X"] (["0"-"9","a"-"f","A"-"F"])+> |
+        <#OCTAL: "0" (["0"-"7"])*>
+}
+
+TOKEN :
+{
+    <AUTOMATADIRECTIVE: "@automata"> |
+    <COLON: ":"> |
+    <COMMA: ","> |
+    <CONDITION: ":-"> |
+    <CONTAINS: "=~"> |
+    <DASH: "-"> |
+    <DEFAULTDIRECTIVE: "@default"> |
+    <DIFFERENT: "!="> |
+    <DOLLAR: "$"> |
+    <DOT: "."> |
+    <ELLIPSIS: "..."> |
+    <EQUALS: "="> |
+    <EXCLAMATION: "!"> |
+    <INCLUDEDIRECTIVE: "@include"> |
+    <LARGER: ">"> |
+    <LARGEREQUALS: ">="> |
+    <LEFTBRACE: "("> |
+    <LEFTSQUAREBRACKET: "["> |
+    <LITERAL: "'" (~["'"] | "\\'")* "'"> |
+    <NL: "\n"> |
+    <PLUS: "+"> |
+    <PRODUCE: "+>"> |
+    <QUESTION: "?"> |
+    <QUOTE: "\""> |
+    <REPLACE: "->"> |
+    <RIGHTBRACE: ")"> |
+    <RIGHTSQUAREBRACKET: "]"> |
+    <SEMICOLON: ";"> |
+    <SLASH: "/"> |
+    <SMALLER: "<"> |
+    <SMALLEREQUALS: "<="> |
+    <STEMMINGDIRECTIVE: "@stemming"> |
+    <SUPERDIRECTIVE: "@super"> |
+    <IDENTIFIER: (~[
+"\u0000"-"\u002f","\u003a"-"\u003f","\u005b"-"\u005d","\u007b"-"\u00a7","\u00a9","\u00ab"-"\u00ae","\u00b0"-"\u00b3","\u00b6"-"\u00b7","\u00b9","\u00bb"-"\u00bf",
+"\u00d7","\u00f7","\u037e","\u0387","\u03f6","\u0482","\u055a"-"\u055f","\u0589"-"\u058a","\u05be","\u05c0","\u05c3","\u05c6","\u05f3"-"\u05f4","\u0600"-"\u0603",
+"\u0606"-"\u060f","\u061b","\u061e"-"\u061f","\u066a"-"\u066d","\u06d4","\u06dd"-"\u06de","\u06e9","\u06fd"-"\u06fe","\u0700"-"\u070d","\u070f","\u07f6"-"\u07f9",
+"\u0830"-"\u083e","\u085e","\u0964"-"\u0965","\u0970","\u09f2"-"\u09fb","\u0af1","\u0b70","\u0b72"-"\u0b77","\u0bf0"-"\u0bfa","\u0c78"-"\u0c7f","\u0d70"-"\u0d75",
+"\u0d79","\u0df4","\u0e3f","\u0e4f","\u0e5a"-"\u0e5b","\u0f01"-"\u0f17","\u0f1a"-"\u0f1f","\u0f2a"-"\u0f34","\u0f36","\u0f38","\u0f3a"-"\u0f3d","\u0f85",
+"\u0fbe"-"\u0fc5","\u0fc7"-"\u0fcc","\u0fce"-"\u0fda","\u104a"-"\u104f","\u109e"-"\u109f","\u10fb","\u1360"-"\u137c","\u1390"-"\u1399","\u1400","\u166d"-"\u166e",
+"\u1680","\u169b"-"\u169c","\u16eb"-"\u16f0","\u1735"-"\u1736","\u17b4"-"\u17b5","\u17d4"-"\u17d6","\u17d8"-"\u17db","\u17f0"-"\u17f9","\u1800"-"\u180a",
+"\u180e","\u1940","\u1944"-"\u1945","\u19da","\u19de"-"\u19ff","\u1a1e"-"\u1a1f","\u1aa0"-"\u1aa6","\u1aa8"-"\u1aad","\u1b5a"-"\u1b6a","\u1b74"-"\u1b7c",
+"\u1bfc"-"\u1bff","\u1c3b"-"\u1c3f","\u1c7e"-"\u1c7f","\u1cd3","\u2000"-"\u2064","\u206a"-"\u2070","\u2074"-"\u207e","\u2080"-"\u208e","\u20a0"-"\u20b9",
+"\u2100"-"\u2101","\u2103"-"\u2106","\u2108"-"\u2109","\u2114","\u2116"-"\u2118","\u211e"-"\u2123","\u2125","\u2127","\u2129","\u212e","\u213a"-"\u213b",
+"\u2140"-"\u2144","\u214a"-"\u214d","\u214f"-"\u2182","\u2185"-"\u2189","\u2190"-"\u23f3","\u2400"-"\u2426","\u2440"-"\u244a","\u2460"-"\u26ff","\u2701"-"\u27ca",
+"\u27cc","\u27ce"-"\u2b4c","\u2b50"-"\u2b59","\u2ce5"-"\u2cea","\u2cf9"-"\u2cff","\u2d70","\u2e00"-"\u2e2e","\u2e30"-"\u2e31","\u2e80"-"\u2e99","\u2e9b"-"\u2ef3",
+"\u2f00"-"\u2fd5","\u2ff0"-"\u2ffb","\u3000"-"\u3004","\u3007"-"\u3029","\u3030","\u3036"-"\u303a","\u303d"-"\u303f","\u30a0","\u30fb","\u3190"-"\u319f",
+"\u31c0"-"\u31e3","\u3200"-"\u321e","\u3220"-"\u32fe","\u3300"-"\u33ff","\u4dc0"-"\u4dff","\ua490"-"\ua4c6","\ua4fe"-"\ua4ff","\ua60d"-"\ua60f","\ua673",
+"\ua67e","\ua6e6"-"\ua6ef","\ua6f2"-"\ua6f7","\ua828"-"\ua82b","\ua830"-"\ua839","\ua874"-"\ua877","\ua8ce"-"\ua8cf","\ua8f8"-"\ua8fa","\ua92e"-"\ua92f",
+"\ua95f","\ua9c1"-"\ua9cd","\ua9de"-"\ua9df","\uaa5c"-"\uaa5f","\uaa77"-"\uaa79","\uaade"-"\uaadf","\uabeb","\ue000"-"\uf8ff","\ufb29","\ufd3e"-"\ufd3f",
+"\ufdfc"-"\ufdfd","\ufe10"-"\ufe19","\ufe30"-"\ufe52","\ufe54"-"\ufe66","\ufe68"-"\ufe6b","\ufeff","\uff01"-"\uff0f","\uff1a"-"\uff20","\uff3b"-"\uff3d",
+"\uff3f","\uff5b"-"\uff65","\uffe0"-"\uffe2","\uffe4"-"\uffe6","\uffe8"-"\uffee","\ufff9"-"\ufffd"
+    ])+>
+}
+
+/** Parses a search definition and returns the resulting object */
+RuleBase semanticRules(RuleBase rules,RuleImporter importer) :
+{
+}
+{
+    ( LOOKAHEAD(2) <NL> |
+      directive(rules,importer) |
+      LOOKAHEAD(4) namedCondition(rules) |
+      productionRule(rules) ) *
+    { return rules; }
+}
+
+// ---------------------------------- Directive ---------------------------------------
+
+RuleBase directive(RuleBase rules,RuleImporter importer) :
+{
+    String name;
+}
+{
+    ( includeDirective(rules,importer) | defaultDirective(rules) | automataDirective(rules,importer) | stemmingDirective(rules) )
+    { return rules; }
+}
+
+void includeDirective(RuleBase rules,RuleImporter importer) :
+{
+    String name;
+}
+{
+    <INCLUDEDIRECTIVE> <LEFTBRACE> name=stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)?
+    {
+        try {
+            importer.include(name,rules);
+        }
+        catch (java.io.IOException e) {
+            ParseException ep=new ParseException("Could not read included rule base '" +
+                                                 name + "'");
+            ep.initCause(e);
+            throw ep;
+        }
+    }
+}
+
+void automataDirective(RuleBase rules,RuleImporter importer) :
+{
+    String name;
+}
+{
+    <AUTOMATADIRECTIVE> <LEFTBRACE> name=stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)?
+    {
+        importer.setAutomata(rules,name);
+    }
+}
+
+void defaultDirective(RuleBase rules) :
+{
+}
+{
+    <DEFAULTDIRECTIVE> (<SEMICOLON>)?
+    {
+        rules.setDefault(true);
+    }
+}
+
+void stemmingDirective(RuleBase rules) :
+{
+    String booleanString;
+}
+{
+    <STEMMINGDIRECTIVE> <LEFTBRACE> booleanString=stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)?
+    {
+        rules.setStemming(Boolean.parseBoolean(booleanString));
+    }
+}
+
+// ---------------------------------- Production rules --------------------------------
+
+void productionRule(RuleBase rules) :
+{
+    ProductionRule rule;
+    Condition condition;
+    ProductionList production=null;
+}
+{
+    condition=topLevelCondition() rule=productionRuleType() ( production=productionList() )? <SEMICOLON>
+    {
+        rule.setCondition(condition);
+        if (production!=null) rule.setProduction(production);
+        rules.addRule(rule);
+    }
+}
+
+ProductionRule productionRuleType() :
+{
+}
+{
+    ( <REPLACE> { return new ReplacingProductionRule(); } ) |
+    ( <PRODUCE> { return new AddingProductionRule(); } )
+}
+
+ProductionList productionList() :
+{
+    ProductionList productionList=new ProductionList();
+    Production production;
+    int weight=100;
+}
+{
+    ( production=production() (<EXCLAMATION> weight=number())?
+      {
+          production.setWeight(weight);
+          productionList.addProduction(production);
+          weight=100;
+      } (<NL>)*
+    ) +
+    { return productionList; }
+}
+
+Production production() :
+{
+    Production production;
+}
+{
+    ( LOOKAHEAD(2) production=namespaceProduction() | production=termProduction() )
+    { return production; }
+}
+
+TermProduction termProduction() :
+{
+    TermProduction termProduction;
+    TermType termType;
+    String label=null;
+}
+{
+    termType=termType()
+    ( LOOKAHEAD(2) label=label() )?
+    ( termProduction=nonphraseTermProduction() | termProduction=phraseProduction() )
+
+    {
+        termProduction.setLabel(label);
+        termProduction.setTermType(termType);
+        return termProduction;
+    }
+}
+
+TermProduction nonphraseTermProduction() :
+{
+    TermProduction termProduction;
+}
+{
+    ( termProduction=referenceTermProduction() |
+      termProduction=literalTermProduction() )
+    {
+      return termProduction;
+    }
+}
+
+LiteralPhraseProduction phraseProduction() :
+{
+    LiteralPhraseProduction phraseProduction=new LiteralPhraseProduction();
+    String term=null;
+}
+{
+
+    <QUOTE>
+    (
+      term=identifier()
+      { phraseProduction.addTerm(term); }
+    )+
+    <QUOTE>
+
+    { return phraseProduction; }
+
+}
+
+NamespaceProduction namespaceProduction() :
+{
+    String namespace;
+    String key;
+    String value=null;
+}
+{
+    namespace=identifier() <DOT> key=stringOrLiteral() <EQUALS> value=identifierOrLiteral()
+    { return new NamespaceProduction(namespace,key,value); }
+}
+
+ReferenceTermProduction referenceTermProduction() :
+{
+    String reference;
+}
+{
+    <LEFTSQUAREBRACKET> reference=referenceIdentifier() <RIGHTSQUAREBRACKET>
+    { return new ReferenceTermProduction(reference); }
+}
+
+LiteralTermProduction literalTermProduction() :
+{
+    String literal;
+}
+{
+    literal=identifier()
+    { return new LiteralTermProduction(literal); }
+}
+
+TermType termType() :
+{
+}
+{
+    <QUESTION> { return TermType.OR; } |
+    <DOLLAR> { return TermType.RANK; } |
+    <PLUS> { return TermType.AND; }  |
+    <DASH> { return TermType.NOT; } |
+    { return TermType.DEFAULT; }
+}
+
+String referenceIdentifier() :
+{
+    String reference;
+}
+{
+    ( reference=identifier() { return reference; } )
+    |
+    ( <ELLIPSIS> { return "..."; } )
+}
+
+// ---------------------------------- Conditions -------------------------------------
+
+void namedCondition(RuleBase rules) :
+{
+    String conditionName;
+    Condition condition;
+}
+{
+    <LEFTSQUAREBRACKET> conditionName=identifier() <RIGHTSQUAREBRACKET> <CONDITION> condition=topLevelCondition() <SEMICOLON>
+    { rules.addCondition(new NamedCondition(conditionName,condition)); }
+}
+
+Condition topLevelCondition() :
+{
+    Condition condition;
+    boolean startAnchor=false;
+    boolean endAnchor=false;
+}
+{
+    ( <DOT> { startAnchor=true; } )?
+    (
+      LOOKAHEAD(3) condition=choiceCondition() |
+      LOOKAHEAD(3) condition=sequenceCondition()
+    )
+    ( LOOKAHEAD(2) <DOT> { endAnchor=true; } )?
+    {
+      condition.setAnchor(Condition.Anchor.create(startAnchor,endAnchor));
+      return condition;
+    }
+}
+
+Condition condition() :
+{
+    Condition condition;
+}
+{
+  (
+    ( LOOKAHEAD(3) condition=choiceCondition()
+      | condition=terminalCondition() )
+    {
+      return condition;
+    }
+  )
+}
+
+Condition terminalOrSequenceCondition() :
+{
+    Condition condition;
+}
+{
+    ( LOOKAHEAD(3) condition=sequenceCondition() |
+      condition=terminalCondition() )
+    { return condition; }
+}
+
+Condition terminalCondition() :
+{
+    Condition condition;
+}
+{
+    ( condition=notCondition() | condition=terminalOrComparisonCondition() )
+    { return condition; }
+}
+
+Condition terminalOrComparisonCondition() :
+{
+    Condition condition,rightCondition;
+    String comparison;
+}
+{
+    condition=reallyTerminalCondition()
+    ( comparison=comparison() ( LOOKAHEAD(2) rightCondition=nestedCondition() | rightCondition=reallyTerminalCondition() )
+//    ( comparison=comparison() rightCondition=condition()
+      { condition=new ComparisonCondition(condition,comparison,rightCondition); }
+    ) ?
+    { return condition; }
+
+}
+
+Condition reallyTerminalCondition() :
+{
+    String label=null;
+    String context=null;
+    String nameSpace=null;
+    Condition condition=null;
+}
+{
+// This body looks like this to distinguish these two cases
+// namespace.condition
+// condition .  (end anchor)
+   ( LOOKAHEAD(8)
+   (
+       ( LOOKAHEAD(2) context=context() )?
+       ( nameSpace=nameSpace() )
+       ( LOOKAHEAD(2) label=label() )?
+       condition=terminalConditionBody()
+   )
+   |
+   (
+       ( LOOKAHEAD(2) context=context() )?
+       ( LOOKAHEAD(2) label=label() )?
+       condition=terminalConditionBody()
+   )
+   )
+   {
+     if (context!=null)
+         condition.setContextName(context);
+     condition.setLabel(label);
+     condition.setNameSpace(nameSpace);
+     return condition;
+   }
+}
+
+
+Condition terminalConditionBody() :
+{
+    Condition condition=null;
+}
+{
+    (
+    LOOKAHEAD(2) condition=conditionReference() |
+    condition=termCondition() |
+    condition=nestedCondition() |
+    condition=nonReferableEllipsisCondition() |
+    condition=referableEllipsisCondition() |
+    condition=superCondition() |
+    condition=literalCondition() |
+    condition=compositeItemCondition())
+    { return condition; }
+}
+
+Condition notCondition() :
+{
+    Condition condition;
+}
+{
+    <EXCLAMATION> condition=terminalOrComparisonCondition()
+    { return new NotCondition(condition); }
+}
+
+
+ConditionReference conditionReference() :
+{
+    String conditionName;
+}
+{
+    <LEFTSQUAREBRACKET> conditionName=identifier() <RIGHTSQUAREBRACKET>
+    { return new ConditionReference(conditionName); }
+}
+
+EllipsisCondition nonReferableEllipsisCondition() :
+{
+}
+{
+    <ELLIPSIS> { return new EllipsisCondition(false); }
+}
+
+EllipsisCondition referableEllipsisCondition() :
+{
+}
+{
+    <LEFTSQUAREBRACKET> <ELLIPSIS> <RIGHTSQUAREBRACKET>
+    { return new EllipsisCondition(); }
+}
+
+Condition nestedCondition() :
+{
+    Condition condition;
+}
+{
+    <LEFTBRACE> condition=choiceCondition() <RIGHTBRACE>
+    { return condition; }
+}
+
+Condition sequenceCondition() :
+{
+    SequenceCondition sequenceCondition=new SequenceCondition();
+    Condition condition;
+}
+{
+    condition=terminalCondition()
+    { sequenceCondition.addCondition(condition); }
+    ( LOOKAHEAD(2) condition=terminalCondition()
+      { sequenceCondition.addCondition(condition); }
+    )*
+    {
+      if (sequenceCondition.conditionSize()==1)
+          return sequenceCondition.removeCondition(0);
+      else
+          return sequenceCondition;
+    }
+}
+
+Condition choiceCondition() :
+{
+    ChoiceCondition choiceCondition=new ChoiceCondition();
+    Condition condition;
+}
+{
+    condition=terminalOrSequenceCondition()
+    { choiceCondition.addCondition(condition); }
+    ( LOOKAHEAD(3) (<NL>)* <COMMA> (<NL>)* condition=terminalOrSequenceCondition()
+      { choiceCondition.addCondition(condition); }
+    ) *
+    {
+      if (choiceCondition.conditionSize()==1)
+          return choiceCondition.removeCondition(0);
+      else
+          return choiceCondition;
+    }
+}
+
+TermCondition termCondition() :
+{
+    String str;
+}
+{
+    ( str = identifier() )
+    { return new TermCondition(str); }
+}
+
+SuperCondition superCondition() : { }
+{
+    ( <SUPERDIRECTIVE> )
+    { return new SuperCondition(); }
+}
+
+LiteralCondition literalCondition() :
+{
+    String str;
+}
+{
+    ( str = literal() )
+    { return new LiteralCondition(str); }
+}
+
+CompositeItemCondition compositeItemCondition() :
+{
+    Condition condition;
+    CompositeItemCondition compositeItemCondition = new CompositeItemCondition();
+}
+{
+    ( <QUOTE> ( condition=terminalConditionBody() { compositeItemCondition.addCondition(condition); } ) <QUOTE> )
+    { return compositeItemCondition; }
+}
+
+// ---------------------------------- Primitives -------------------------------------
+
+String context() :
+{
+    String str;
+}
+{
+    ( str = identifier() <SLASH> )
+    { return str; }
+}
+
+String label() :
+{
+    String str;
+}
+{
+    ( str = identifier() <COLON> )
+    { return str; }
+}
+
+String nameSpace() :
+{
+    String str;
+}
+{
+    ( str = identifier() <DOT> )
+    { return str; }
+}
+
+String identifier() : { }
+{
+    ( <IDENTIFIER> | <NUMBER> )
+    { return token.image; }
+}
+
+String stringOrLiteral() :
+{
+    String str;
+}
+{
+    ( str = string() | str = literal() )
+    { return str; }
+}
+
+String identifierOrLiteral() :
+{
+    String str;
+}
+{
+    ( str = identifier() | str = literal() )
+    { return str; }
+}
+
+String comparison() : { }
+{
+    ( <DIFFERENT> | <CONTAINS> | <EQUALS> | <LARGEREQUALS> | <SMALLEREQUALS> | <LARGER> | <SMALLER> )
+    { return token.image; }
+}
+
+String string() :
+{
+    StringBuilder str = new StringBuilder();
+}
+{
+    ( ( <SLASH> | <IDENTIFIER> | <DOT> | <DASH> ) { str.append(token.image); } ) +
+    { return str.toString(); }
+}
+
+String literal() : { }
+{
+    ( <LITERAL> )
+    { return UnicodeUtilities.unquote(token.image); }
+}
+
+int number() : { }
+{
+    <NUMBER> { return Integer.decode(token.image); }
+}
diff --git a/container-search/src/main/javacc/com/yahoo/search/grouping/request/parser/GroupingParser.jj b/container-search/src/main/javacc/com/yahoo/search/grouping/request/parser/GroupingParser.jj
new file mode 100644
index 00000000000..89433ab52f5
--- /dev/null
+++ b/container-search/src/main/javacc/com/yahoo/search/grouping/request/parser/GroupingParser.jj
@@ -0,0 +1,1148 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// --------------------------------------------------------------------------------
+//
+// JavaCC options. When this file is changed, run "ant compileparser" to rebuild
+// the parser classes.
+//
+// --------------------------------------------------------------------------------
+options {
+    CACHE_TOKENS       = true;
+    STATIC             = false;
+    DEBUG_PARSER       = false;
+    USER_TOKEN_MANAGER = false;
+    USER_CHAR_STREAM   = true;
+    ERROR_REPORTING    = true;
+}
+
+// --------------------------------------------------------------------------------
+//
+// Parser body.
+//
+// --------------------------------------------------------------------------------
+PARSER_BEGIN(GroupingParser)
+
+package com.yahoo.search.grouping.request.parser;
+
+import java.util.List;
+import java.util.LinkedList;
+import com.yahoo.javacc.UnicodeUtilities;
+import com.yahoo.search.grouping.request.*;
+
+@SuppressWarnings({ "ConstantIfStatement", "UnnecessarySemicolon", "Convert2Diamond", "FinalPrivateMethod" })
+public class GroupingParser {
+
+    private static Number resolveNumber(String str) {
+        if (str.indexOf('.') >= 0) {
+            return Double.valueOf(str);
+        }
+        return Long.valueOf(str);
+    }
+}
+
+PARSER_END(GroupingParser)
+        
+TOKEN :
+{
+    <INTEGER: <DECIMAL> (["l","L"])? | <HEX> (["l","L"])? | <OCTAL> (["l","L"])?> |
+        <#DECIMAL: ["1"-"9"] (["0"-"9"])*> |
+        <#HEX: "0" ["x","X"] (["0"-"9","a"-"f","A"-"F"])+> |
+        <#OCTAL: "0" (["0"-"7"])*> |
+    <FLOAT: (["0"-"9"])+ ("." (["0"-"9"])*)? (<EXPONENT>)? (["f","F","d","D"])?> |  
+        <#EXPONENT: ["e","E"] (["+","-"])? (["0"-"9"])+>
+}
+
+//
+// NOTE: Whenever you add a new token, you need to also add it do the identifier() rule at the bottom of this grammar.
+// NOTE: Failure to do so will cause the token image to become a reserved word. To prevent regression on this, add the
+// NOTE: new token to the GroupingParserTestCase#requireThatTokenImagesAreNotReservedWords() test.
+//
+TOKEN :
+{
+    <SPACE: [" ","\n","\r","\t"]> |
+    <SCOLON: ";"> |
+    <COMMA: ","> |
+    <DOLLAR: "$"> |
+    <DOT: "."> |
+    <EQ: "="> |
+    <INFIX_ADD: "+"> |
+    <INFIX_DIV: "/"> |
+    <INFIX_MOD: "%"> |
+    <INFIX_MUL: "*"> |
+    <INFIX_SUB: "-"> |
+    <LBRACE: "("> |
+    <RBRACE: ")"> |
+    <LCURLY: "{"> |
+    <RCURLY: "}"> |
+    <LT: "<"> |
+    <GT: ">"> |
+    <LBRACKET: "["> |
+    <RBRACKET: "]"> |
+    <INF: "inf"> |
+    <NEGINF: "-inf"> |
+    <STRING: ("\"" (~["\""] | "\\\"")* "\"") | 
+             ("'" (~["'"] | "\\'")* "'")> |
+    <ACOS: "acos"> |
+    <ACOSH: "acosh"> |
+    <ACCURACY: "accuracy"> |
+    <ADD: "add"> |
+    <ALIAS: "alias"> |
+    <ALL: "all"> |
+    <AND: "and"> |
+    <ARRAY: "array"> |
+    <AS: "as"> |
+    <AT: "at"> |
+    <ASIN: "asin"> |
+    <ASINH: "asinh"> |
+    <ATAN: "atan"> |
+    <ATANH: "atanh"> |
+    <ATTRIBUTE: "attribute"> |
+    <AVG: "avg"> |
+    <BUCKET: "bucket"> |
+    <CAT: "cat"> |
+    <CBRT: "cbrt"> |
+    <COS: "cos"> |
+    <COSH: "cosh"> |
+    <COUNT: "count"> |
+    <DEBUGWAIT: "debugwait"> |
+    <DIV: "div"> |
+    <DOCIDNSSPECIFIC: "docidnsspecific"> |
+    <EACH: "each"> |
+    <EXP: "exp"> |
+    <FIXEDWIDTH: "fixedwidth"> |
+    <FLOOR: "floor"> |
+    <GROUP: "group"> |
+    <HINT: "hint"> |
+    <HYPOT: "hypot"> |
+    <LOG: "log"> |
+    <LOG1P: "log1p"> |
+    <LOG10: "log10"> |
+    <MATH: "math"> |
+    <MAX: "max"> |
+    <MD5: "md5"> |
+    <MIN: "min"> |
+    <MOD: "mod"> |
+    <MUL: "mul"> |
+    <NEG: "neg"> |
+    <NORMALIZESUBJECT: "normalizesubject"> |
+    <NOW: "now"> |
+    <OR: "or"> |
+    <ORDER: "order"> |
+    <OUTPUT: "output"> |
+    <POW: "pow"> |
+    <PRECISION: "precision"> |
+    <PREDEFINED: "predefined"> |
+    <RELEVANCE: "relevance"> |
+    <REVERSE: "reverse"> |
+    <SIN: "sin"> |
+    <SINH: "sinh"> |
+    <SIZE: "size"> |
+    <SORT: "sort"> |
+    <INTERPOLATEDLOOKUP: "interpolatedlookup"> |
+    <SQRT: "sqrt"> |
+    <STRCAT: "strcat"> |
+    <STRLEN: "strlen"> |
+    <SUB: "sub"> |
+    <SUM: "sum"> |
+    <SUMMARY: "summary"> |
+    <TAN: "tan"> |
+    <TANH: "tanh"> |
+    <TIME: "time"> |
+    <TIME_DATE: "date"> |
+    <TIME_DAYOFMONTH: "dayofmonth"> |
+    <TIME_DAYOFWEEK: "dayofweek"> |
+    <TIME_DAYOFYEAR: "dayofyear"> |
+    <TIME_HOUROFDAY: "hourofday"> |
+    <TIME_MINUTEOFHOUR: "minuteofhour"> |
+    <TIME_MONTHOFYEAR: "monthofyear"> |
+    <TIME_SECONDOFMINUTE: "secondofminute"> |
+    <TIME_YEAR: "year"> |
+    <TODOUBLE: "todouble"> |
+    <TOLONG: "tolong"> |
+    <TORAW: "toraw"> |
+    <TOSTRING: "tostring"> |
+    <TRUE: "true"> |
+    <FALSE: "false"> |
+    <UCA: "uca"> |
+    <WHERE: "where"> |
+    <X: "x"> |
+    <XOR: "xor"> |
+    <XORBIT: "xorbit"> |
+    <Y: "y"> |
+    <YMUM: "ymum"> |
+    <ZCURVE: "zcurve"> |
+    <IDENTIFIER: ["A"-"Z","a"-"z"](["A"-"Z","a"-"z","0"-"9","_","@"])*>
+}
+
+// --------------------------------------------------------------------------------
+//
+// Production rules.
+//
+// --------------------------------------------------------------------------------
+
+List<GroupingOperation> requestList() :
+{
+    List<GroupingOperation> lst = new LinkedList<GroupingOperation>();
+    GroupingOperation val;
+}
+{
+    ( space() val = root() { lst.add(val); } ( scolon() val = root() { lst.add(val); } )* <EOF> )
+    { return lst;}
+}
+
+GroupingOperation request() :
+{
+    GroupingOperation val;
+}
+{
+    ( space() val = root() <EOF> )
+    { return val; }
+}
+
+GroupingOperation root() :
+{
+    String str;
+    GroupingOperation ret;
+}
+{
+    ( ret = operation(null)
+      [ <WHERE> lbrace() str = where() rbrace() { GroupingOperation grp = new AllOperation();
+                                                  grp.setWhere(str);
+                                                  grp.addChild(ret);
+                                                  ret = grp; } ] )
+    { return ret; }
+}
+
+GroupingOperation operation(GroupingOperation parent) :
+{
+    GroupingOperation ret;
+}
+{
+    ( ret = allOperation(parent) | ret = eachOperation(parent) )
+    { return ret; }
+}
+
+GroupingOperation allOperation(GroupingOperation parent) :
+{
+    GroupingOperation ret;
+}
+{
+    ( <ALL> { ret = new AllOperation(); if (parent != null) { parent.addChild(ret); } }
+      lbrace() operationBody(ret) rbrace() )
+    { return ret; }
+}
+
+GroupingOperation eachOperation(GroupingOperation parent) :
+{
+    String str;
+    GroupingOperation ret;
+}
+{
+    ( <EACH> { ret = new EachOperation(); if (parent != null) { parent.addChild(ret); } }
+      lbrace() operationBody(ret) rbrace()
+      [ <AS> lbrace() str = string() rbrace() { ret.setLabel(str); } ] )
+    { return ret; }
+}
+
+void operationBody(GroupingOperation parent) :
+{
+    String str;
+    Number num;
+    GroupingExpression exp;
+    List<GroupingExpression> lst;
+}
+{
+    [   <GROUP>     lbrace() exp = exp(parent)                            rbrace() { parent.setGroupBy(exp); }                  ]
+    ( ( <ACCURACY>  lbrace() num = number()                               rbrace() { parent.setAccuracy(num.doubleValue()); } ) |
+      ( <ALIAS>     lbrace() str = identifier() comma() exp = exp(parent) rbrace() { parent.putAlias(str, exp); }             ) |
+      ( <HINT>      lbrace() str = identifier()                           rbrace() { parent.addHint(str); }                   ) |
+      ( <MAX>       lbrace() num = number()                               rbrace() { parent.setMax(num.intValue()); }         ) |
+      ( <ORDER>     lbrace() lst = expList(parent)                        rbrace() { parent.addOrderBy(lst); }                ) |
+      ( <OUTPUT>    lbrace() lst = expList(parent)                        rbrace() { parent.addOutputs(lst); }                ) |
+      ( <PRECISION> lbrace() num = number()                               rbrace() { parent.setPrecision(num.intValue()); }   ) |
+      ( <WHERE>     lbrace() str = where()                                rbrace() { parent.setWhere(str); }                  ) )*
+    (                              operation(parent)                                                                            )*
+}
+
+String where() :
+{
+    StringBuilder ret = new StringBuilder();
+    String str;
+}
+{
+    ( [ <DOLLAR> { ret.append(token.image); } ] str = identifier() { ret.append(str); } )
+    { return ret.toString(); }
+}
+
+List<GroupingExpression> expList(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+    List<GroupingExpression> ret = new LinkedList<GroupingExpression>();
+}
+{
+    ( exp = exp(grp) { ret.add(exp); } ( comma() exp = exp(grp) { ret.add(exp); } )* )
+    { return ret; }
+}
+
+GroupingExpression exp(GroupingOperation grp) :
+{
+    String str;
+    boolean neg = false;
+    GroupingExpression exp;
+}
+{
+    ( [ add() | sub()                         { neg = true; } ]
+      exp = expAlias(grp)                     { if (neg) { exp = new NegFunction(exp); } }
+      [ <AS> lbrace() str = string() rbrace() { exp.setLabel(str); } ] )
+    { return exp; }
+}
+
+GroupingExpression expAlias(GroupingOperation grp) :
+{
+    String str;
+    GroupingExpression exp;
+}
+{
+    ( ( <DOLLAR> str = identifier() [ eq() exp = expMath(grp) { grp.putAlias(str, exp); } ] { exp = grp.getAlias(str); } ) |
+      ( exp = expMath(grp) ) )
+    { return exp; }
+}
+
+GroupingExpression expMath(GroupingOperation grp) :
+{
+    MathResolver.Type type = MathResolver.Type.ADD;
+    MathResolver math = new MathResolver();
+    GroupingExpression exp;
+}
+{
+    ( exp = value(grp) { math.push(type, exp); }
+      ( ( add() { type = MathResolver.Type.ADD; } |
+          div() { type = MathResolver.Type.DIV; } |
+          mod() { type = MathResolver.Type.MOD; } |
+          mul() { type = MathResolver.Type.MUL; } |
+          sub() { type = MathResolver.Type.SUB; } ) exp = value(grp) { math.push(type, exp); } )* )
+    { return math.resolve(); }
+}
+
+GroupingExpression value(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+}
+{
+    ( LOOKAHEAD(2)
+      ( ( lbrace() exp = exp(grp) rbrace() )           |
+                   exp = addFunction(grp)              |
+                   exp = andFunction(grp)              |
+                   exp = attributeFunction()           |
+                   exp = avgExpression(grp)            |
+                   exp = catFunction(grp)              |
+                   exp = constantValue()               |
+                   exp = countAggregator()             |
+                   exp = debugWaitFunction(grp)        |
+                   exp = divFunction(grp)              |
+                   exp = docIdNsSpecificValue()        |
+                   exp = fixedWidthFunction(grp)       |
+                   exp = mathFunction(grp)             |
+                   exp = maxExpression(grp)            |
+                   exp = md5Function(grp)              |
+                   exp = minExpression(grp)            |
+                   exp = modFunction(grp)              |
+                   exp = mulFunction(grp)              |
+                   exp = negFunction(grp)              |
+                   exp = normalizeSubjectFunction(grp) |
+                   exp = nowFunction()                 |
+                   exp = orFunction(grp)               |
+                   exp = predefinedFunction(grp)       |
+                   exp = relevanceValue()              |
+                   exp = reverseFunction(grp)          |
+                   exp = sizeFunction(grp)             |
+                   exp = sortFunction(grp)             |
+                   exp = arrayAtLookup(grp)            |
+                   exp = interpolatedLookup(grp)       |
+                   exp = stringValue()                 |
+                   exp = strCatFunction(grp)           |
+                   exp = strLenFunction(grp)           |
+                   exp = subFunction(grp)              |
+                   exp = sumAggregator(grp)            |
+                   exp = summaryValue()                |
+                   exp = timeFunction(grp)             |
+                   exp = toDoubleFunction(grp)         |
+                   exp = toLongFunction(grp)           |
+                   exp = toRawFunction(grp)            |
+                   exp = toStringFunction(grp)         |
+                   exp = ucaFunction(grp)              |
+                   exp = xorExpression(grp)            |
+                   exp = xorBitFunction(grp)           |
+                   exp = ymumValue()                   |
+                   exp = zcurveFunction(grp)           ) |
+      (            exp = attributeValue()              ) )
+    { return exp; }
+}
+
+AddFunction addFunction(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2;
+}
+{
+    ( <ADD> lbrace() arg1 = exp(grp) comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] rbrace() )
+    { if (argN == null) return new AddFunction(arg1, arg2);
+      return new AddFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+AndFunction andFunction(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2;
+}
+{
+    ( <AND> lbrace() arg1 = exp(grp) comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] rbrace() )
+    { if (argN == null) return new AndFunction(arg1, arg2);
+      return new AndFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+AttributeValue attributeValue() :
+{
+    StringBuilder ret = new StringBuilder();
+    String str;
+}
+{
+    ( str = identifier() { ret.append(str); }
+    ( ( <DOT> { ret.append(token.image); } ( str = identifier() { ret.append(str); } ) ) |
+      ( lcurly() str = string() { ret.append("{\"").append(str).append("\"}"); } rcurly() ) )* )
+    { return new AttributeValue(ret.toString()); }
+}
+
+AttributeFunction attributeFunction() :
+{
+    AttributeValue val;
+}
+{
+    ( <ATTRIBUTE> lbrace() val = attributeValue() rbrace() )
+    { return new AttributeFunction(val.getAttributeName()); }
+}
+
+ArrayAtLookup arrayAtLookup(GroupingOperation parent) :
+{
+    AttributeValue val = null;
+    GroupingExpression arg2 = null;
+}
+{
+    ( <ARRAY> <DOT> <AT> lbrace() val = attributeValue() comma() arg2 = exp(parent) rbrace() )
+    {
+        return new ArrayAtLookup(val.getAttributeName(), arg2);
+    }
+}
+
+InterpolatedLookup interpolatedLookup(GroupingOperation parent) :
+{
+    AttributeValue val = null;
+    GroupingExpression arg2 = null;
+}
+{
+    ( <INTERPOLATEDLOOKUP> lbrace() val = attributeValue() comma() arg2 = exp(parent) rbrace() )
+    {
+        return new InterpolatedLookup(val.getAttributeName(), arg2);
+    }
+}
+
+GroupingExpression avgExpression(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2 = null;
+}
+{
+    ( <AVG> lbrace() arg1 = exp(grp) [ comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] ] rbrace() )
+    { if (arg2 == null) return new AvgAggregator(arg1);
+      if (argN == null) return new AvgFunction(arg1, arg2);
+      return new AvgFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+CatFunction catFunction(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2;
+}
+{
+    ( <CAT> lbrace() arg1 = exp(grp) comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] rbrace() )
+    { if (argN == null) return new CatFunction(arg1, arg2);
+      return new CatFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+ConstantValue constantValue() :
+{
+    Number num;
+}
+{
+    ( num = number() )
+    { return num instanceof Double ? new DoubleValue((Double)num) : new LongValue((Long)num); }
+}
+
+ConstantValue constantValueSigned() :
+{
+    Number num;
+}
+{
+    ( num = signedNumber() )
+    { return num instanceof Double ? new DoubleValue((Double)num) : new LongValue((Long)num); }
+}
+
+InfiniteValue infiniteNegativeValue() : { }
+{
+    ( neginf() { return new InfiniteValue(new Infinite(true)); } )
+}
+
+InfiniteValue infinitePositiveValue() : { }
+{
+    ( inf() { return new InfiniteValue(new Infinite(false)); } )
+}
+
+CountAggregator countAggregator() : { }
+{
+    ( <COUNT> lbrace() rbrace() )
+    { return new CountAggregator(); }
+}
+
+DebugWaitFunction debugWaitFunction(GroupingOperation grp) :
+{
+	GroupingExpression arg;
+	Number waitTime;
+	BooleanValue busy;
+}
+{   ( <DEBUGWAIT> lbrace() arg = exp(grp) comma()
+                           waitTime = number() comma()
+                           busy = booleanValue()
+                  rbrace() )
+  	{ return new DebugWaitFunction(arg, new DoubleValue(waitTime.doubleValue()), busy); }
+}
+
+DivFunction divFunction(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2;
+}
+{
+    ( <DIV> lbrace() arg1 = exp(grp) comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] rbrace() )
+    { if (argN == null) return new DivFunction(arg1, arg2);
+      return new DivFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+DocIdNsSpecificValue docIdNsSpecificValue() : { }
+{
+    ( <DOCIDNSSPECIFIC> lbrace() rbrace() )
+    { return new DocIdNsSpecificValue(); }
+}
+
+FixedWidthFunction fixedWidthFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+    Number num;
+}
+{
+    ( <FIXEDWIDTH> lbrace() exp = exp(grp) comma() num = number() rbrace() )
+    { return new FixedWidthFunction(exp, num); }
+}
+
+GroupingExpression maxExpression(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2 = null;
+}
+{
+    ( <MAX> lbrace() arg1 = exp(grp) [ comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] ] rbrace() )
+    { if (arg2 == null) return new MaxAggregator(arg1);
+      if (argN == null) return new MaxFunction(arg1, arg2);
+      return new MaxFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+GroupingExpression md5Function(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+    Number num;
+}
+{
+    ( <MD5> lbrace() exp = exp(grp) comma() num = number() rbrace() )
+    { return new Md5Function(exp, num.intValue()); }
+}
+
+GroupingExpression minExpression(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2 = null;
+}
+{
+    ( <MIN> lbrace() arg1 = exp(grp) [ comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] ] rbrace() )
+    { if (arg2 == null) return new MinAggregator(arg1);
+      if (argN == null) return new MinFunction(arg1, arg2);
+      return new MinFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+ModFunction modFunction(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2;
+}
+{
+    ( <MOD> lbrace() arg1 = exp(grp) comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] rbrace() )
+    { if (argN == null) return new ModFunction(arg1, arg2);
+      return new ModFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+MulFunction mulFunction(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2;
+}
+{
+    ( <MUL> lbrace() arg1 = exp(grp) comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] rbrace() )
+    { if (argN == null) return new MulFunction(arg1, arg2);
+      return new MulFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+NegFunction negFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+}
+{
+    ( <NEG> lbrace() exp = exp(grp) rbrace() )
+    { return new NegFunction(exp); }
+}
+
+NormalizeSubjectFunction normalizeSubjectFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+}
+{
+    ( <NORMALIZESUBJECT> lbrace() exp = exp(grp) rbrace() )
+    { return new NormalizeSubjectFunction(exp); }
+}
+
+NowFunction nowFunction() : { }
+{
+    ( <NOW> lbrace() rbrace() )
+    { return new NowFunction(); }
+}
+
+OrFunction orFunction(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2;
+}
+{
+    ( <OR> lbrace() arg1 = exp(grp) comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] rbrace() )
+    { if (argN == null) return new OrFunction(arg1, arg2);
+      return new OrFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+PredefinedFunction predefinedFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+    BucketResolver resolver = new BucketResolver();
+}
+{
+    ( <PREDEFINED> lbrace() exp = exp(grp) comma()
+      bucket(grp, resolver) ( comma() bucket(grp, resolver) )* rbrace() )
+    { return resolver.resolve(exp); }
+}
+
+RelevanceValue relevanceValue() : { }
+{
+    ( <RELEVANCE> lbrace() rbrace() )
+    { return new RelevanceValue(); }
+}
+
+ReverseFunction reverseFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+}
+{
+    ( <REVERSE> lbrace() exp = exp(grp) rbrace() )
+    { return new ReverseFunction(exp); }
+}
+
+SizeFunction sizeFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+}
+{
+    ( <SIZE> lbrace() exp = exp(grp) rbrace() )
+    { return new SizeFunction(exp); }
+}
+
+SortFunction sortFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+}
+{
+    ( <SORT> lbrace() exp = exp(grp) rbrace() )
+    { return new SortFunction(exp); }
+}
+
+StringValue stringValue() :
+{
+    String str;
+}
+{
+    ( <STRING> { str = UnicodeUtilities.unquote(token.image); } space() )
+    { return new StringValue(str); }
+}
+
+RawValue rawValue(GroupingOperation grp) :
+{
+    RawBuffer buffer = new RawBuffer();
+}
+{
+    ( lcurly() byteValue(buffer) ( comma() byteValue(buffer) )* rcurly() )
+    { return new RawValue(buffer); }
+}
+
+StringValue stringValueUnquoted() :
+{
+    String str;
+}
+{
+    ( str = string() )
+    { return new StringValue(str); }
+}
+
+StrCatFunction strCatFunction(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2;
+}
+{
+    ( <STRCAT> lbrace() arg1 = exp(grp) comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] rbrace() )
+    { if (argN == null) return new StrCatFunction(arg1, arg2);
+      return new StrCatFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+StrLenFunction strLenFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+}
+{
+    ( <STRLEN> lbrace() exp = exp(grp) rbrace() )
+    { return new StrLenFunction(exp); }
+}
+
+SubFunction subFunction(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2;
+}
+{
+    ( <SUB> lbrace() arg1 = exp(grp) comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] rbrace() )
+    { if (argN == null) return new SubFunction(arg1, arg2);
+      return new SubFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+SumAggregator sumAggregator(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+}
+{
+    ( <SUM> lbrace() exp = exp(grp) rbrace() )
+    { return new SumAggregator(exp); }
+}
+
+SummaryValue summaryValue() :
+{
+    String str = null;
+}
+{
+    ( <SUMMARY> lbrace() [ str = string() ] rbrace() )
+    { return str == null ? new SummaryValue() : new SummaryValue(str); }
+}
+
+FunctionNode timeFunction(GroupingOperation grp) :
+{
+    TimeFunctions.Type type;
+    GroupingExpression exp;
+}
+{
+    ( <TIME> <DOT> ( <TIME_DATE>           { type = TimeFunctions.Type.DATE;             } |
+                     <TIME_DAYOFMONTH>     { type = TimeFunctions.Type.DAY_OF_MONTH;     } |
+                     <TIME_DAYOFWEEK>      { type = TimeFunctions.Type.DAY_OF_WEEK;      } |
+                     <TIME_DAYOFYEAR>      { type = TimeFunctions.Type.DAY_OF_YEAR;      } |
+                     <TIME_HOUROFDAY>      { type = TimeFunctions.Type.HOUR_OF_DAY;      } |
+                     <TIME_MINUTEOFHOUR>   { type = TimeFunctions.Type.MINUTE_OF_HOUR;   } |
+                     <TIME_MONTHOFYEAR>    { type = TimeFunctions.Type.MONTH_OF_YEAR;    } |
+                     <TIME_SECONDOFMINUTE> { type = TimeFunctions.Type.SECOND_OF_MINUTE; } |
+                     <TIME_YEAR>           { type = TimeFunctions.Type.YEAR;             } )
+      lbrace() exp = exp(grp) rbrace() )
+    { return TimeFunctions.newInstance(type, exp); }
+}
+
+FunctionNode mathFunction(GroupingOperation grp) :
+{
+    GroupingExpression arg1, arg2 = null;
+    MathFunctions.Function func;
+}
+{
+    ( <MATH> <DOT> ( ( ( <POW>   { func = MathFunctions.Function.POW;   } |
+                         <HYPOT> { func = MathFunctions.Function.HYPOT; } ) lbrace() arg1 = exp(grp) <COMMA>
+                                                                                     arg2 = exp(grp) rbrace() ) |
+                     ( ( <EXP>   { func = MathFunctions.Function.EXP;   } |
+                         <LOG>   { func = MathFunctions.Function.LOG;   } |
+                         <LOG1P> { func = MathFunctions.Function.LOG1P; } |
+                         <LOG10> { func = MathFunctions.Function.LOG10; } |
+                         <SIN>   { func = MathFunctions.Function.SIN;   } |
+                         <ASIN>  { func = MathFunctions.Function.ASIN;  } |
+                         <COS>   { func = MathFunctions.Function.COS;   } |
+                         <ACOS>  { func = MathFunctions.Function.ACOS;  } |
+                         <TAN>   { func = MathFunctions.Function.TAN;   } |
+                         <ATAN>  { func = MathFunctions.Function.ATAN;  } |
+                         <SQRT>  { func = MathFunctions.Function.SQRT;  } |
+                         <SINH>  { func = MathFunctions.Function.SINH;  } |
+                         <ASINH> { func = MathFunctions.Function.ASINH; } |
+                         <COSH>  { func = MathFunctions.Function.COSH;  } |
+                         <ACOSH> { func = MathFunctions.Function.ACOSH; } |
+                         <TANH>  { func = MathFunctions.Function.TANH;  } |
+                         <ATANH> { func = MathFunctions.Function.ATANH; } |
+                         <FLOOR> { func = MathFunctions.Function.FLOOR; } |
+                         <CBRT>  { func = MathFunctions.Function.CBRT;  } ) lbrace() arg1 = exp(grp) rbrace() ) ) )
+    { return MathFunctions.newInstance(func, arg1, arg2); }
+}
+
+FunctionNode ucaFunction(GroupingOperation grp) :
+{
+    GroupingExpression arg;
+    String locale;
+    String strength = null;
+}
+{
+    ( <UCA> lbrace() arg = exp(grp) comma() locale = string() [ comma() strength = string() ] rbrace() )
+    { return (strength == null ? new UcaFunction(arg, locale)
+                               : new UcaFunction(arg, locale, strength)); }
+}
+
+FunctionNode zcurveFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+    int dim;
+}
+{
+    ( <ZCURVE> <DOT> ( <X> { dim = 0; } | <Y> { dim = 1; } ) lbrace() exp = exp(grp) rbrace() )
+    { return dim == 0 ? new ZCurveXFunction(exp) : new ZCurveYFunction(exp); }
+}
+
+ToDoubleFunction toDoubleFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+}
+{
+    ( <TODOUBLE> lbrace() exp = exp(grp) rbrace() )
+    { return new ToDoubleFunction(exp); }
+}
+
+ToLongFunction toLongFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+}
+{
+    ( <TOLONG> lbrace() exp = exp(grp) rbrace() )
+    { return new ToLongFunction(exp); }
+}
+
+ToRawFunction toRawFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+}
+{
+    ( <TORAW> lbrace() exp = exp(grp) rbrace() )
+    { return new ToRawFunction(exp); }
+}
+
+
+ToStringFunction toStringFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+}
+{
+    ( <TOSTRING> lbrace() exp = exp(grp) rbrace() )
+    { return new ToStringFunction(exp); }
+}
+
+GroupingExpression xorExpression(GroupingOperation grp) :
+{
+    List<GroupingExpression> argN = null;
+    GroupingExpression arg1, arg2 = null;
+}
+{
+    ( <XOR> lbrace() arg1 = exp(grp) [ comma() arg2 = exp(grp) [ comma() argN = expList(grp) ] ] rbrace() )
+    { if (arg2 == null) return new XorAggregator(arg1);
+      if (argN == null) return new XorFunction(arg1, arg2);
+      return new XorFunction(arg1, arg2, argN.toArray(new GroupingExpression[argN.size()])); }
+}
+
+XorBitFunction xorBitFunction(GroupingOperation grp) :
+{
+    GroupingExpression exp;
+    Number num;
+}
+{
+    ( <XORBIT> lbrace() exp = exp(grp) comma() num = number() rbrace() )
+    { return new XorBitFunction(exp, num.intValue()); }
+}
+
+YmumValue ymumValue() : { }
+{
+    ( <YMUM> lbrace() rbrace() )
+    { return new YmumValue(); }
+}
+
+void bucket(GroupingOperation grp, BucketResolver resolver) :
+{
+    ConstantValue from, to = null;
+    boolean inclusiveFrom, inclusiveTo;
+}
+{
+    ( <BUCKET> ( ( ( lbrace() | lbracket() ) { inclusiveFrom = true; } ) |
+                 ( lt() { inclusiveFrom = false; } )
+               )
+               ( ( from = infiniteNegativeValue() | from = rawValue(grp) | from = constantValueSigned() | from = stringValueUnquoted() )
+                 ( [ comma() ( to = infinitePositiveValue() | to = rawValue(grp) | to = constantValueSigned() | to = stringValueUnquoted() ) ] )
+               )
+               ( ( ( gt() | rbrace() ) { inclusiveTo = false; } ) |
+                 ( rbracket() { inclusiveTo = true; } )
+               )
+    )
+    { resolver.push(from, inclusiveFrom); resolver.push((to == null) ? from : to, inclusiveTo); }
+}
+
+Number signedNumber() :
+{
+    String str;
+    boolean neg = false;
+}
+{
+    ([ <INFIX_SUB> { neg = true; }]( <INTEGER> | <FLOAT> ) { str = neg ? "-" + token.image : token.image; } space() )
+    { return resolveNumber(str); }
+}
+
+
+Number number() :
+{
+    String str;
+}
+{
+    ( ( <INTEGER> | <FLOAT> ) { str = token.image; } space() )
+    { return resolveNumber(str); }
+}
+
+BooleanValue booleanValue() :
+{
+    String str;
+}
+{
+    ( ( <TRUE> | <FALSE> ) { str = token.image; } space() )
+    { return new BooleanValue(Boolean.parseBoolean(str)); }
+}
+
+void byteValue(RawBuffer buffer) : { }
+{
+    ( ( <INTEGER> { buffer.put(Byte.parseByte(token.image)); } ) |
+      ( <STRING> { buffer.put(token.image.getBytes()); } )
+    )
+}
+
+String string() :
+{
+    String ret;
+}
+{
+    ( ret = identifier() | ( <STRING> { ret = UnicodeUtilities.unquote(token.image); } space() ) )
+    { return ret; }
+}
+
+String identifier() : 
+{
+    String ret;
+}
+{
+    ( ( <ACOS> |
+        <ACOSH> |
+        <ACCURACY> |
+        <ADD> |
+        <ALIAS> |
+        <ALL> |
+        <AND> |
+        <ARRAY> |
+        <AS> |
+        <AT> |
+        <ASIN> |
+        <ASINH> |
+        <ATAN> |
+        <ATANH> |
+        <ATTRIBUTE> |
+        <AVG> |
+        <BUCKET> |
+        <CAT> |
+        <CBRT> |
+        <COS> |
+        <COSH> |
+        <COUNT> |
+        <DEBUGWAIT> |
+        <DIV> |
+        <DOCIDNSSPECIFIC> |
+        <EACH> |
+        <EXP> |
+        <FIXEDWIDTH> |
+        <FLOOR> |
+        <GROUP> |
+        <HINT> |
+        <HYPOT> |
+        <IDENTIFIER> |
+        <LOG> |
+        <LOG1P> |
+        <LOG10> |
+        <MATH> |
+        <MAX> |
+        <MD5> |
+        <MIN> |
+        <MOD> |
+        <MUL> |
+        <NEG> |
+        <NORMALIZESUBJECT> |
+        <NOW> |
+        <OR> |
+        <ORDER> |
+        <OUTPUT> |
+        <POW> |
+        <PRECISION> |
+        <PREDEFINED> |
+        <RELEVANCE> |
+        <REVERSE> |
+        <SIN> |
+        <SINH> |
+        <SIZE> |
+        <SORT> |
+        <INTERPOLATEDLOOKUP> |
+        <SQRT> |
+        <STRCAT> |
+        <STRLEN> |
+        <SUB> |
+        <SUM> |
+        <SUMMARY> |
+        <TAN> |
+        <TANH> |
+        <TIME> |
+        <TIME_DATE> |
+        <TIME_DAYOFMONTH> |
+        <TIME_DAYOFWEEK> |
+        <TIME_DAYOFYEAR> |
+        <TIME_HOUROFDAY> |
+        <TIME_MINUTEOFHOUR> |
+        <TIME_MONTHOFYEAR> |
+        <TIME_SECONDOFMINUTE> |
+        <TIME_YEAR> |
+        <TODOUBLE> |
+        <TOLONG> |
+        <TORAW> |
+        <TOSTRING> |
+        <TRUE> |
+        <FALSE> |
+        <UCA> |
+        <WHERE> |
+        <X> |
+        <XOR> |
+        <XORBIT> |
+        <Y> |
+        <YMUM> |
+        <ZCURVE> ) { ret = token.image; } space() )
+    { return ret; }
+}
+
+void add() : { }
+{
+    ( <INFIX_ADD> space() )
+}
+
+void comma() : { }
+{
+    ( <COMMA> space() )
+}
+
+void div() : { }
+{
+    ( <INFIX_DIV> space() )
+}
+
+void eq() : { }
+{
+    ( <EQ> space() )
+}
+
+void lt() : { }
+{
+    ( <LT> space() )
+}
+
+void gt() : { }
+{
+    ( <GT> space() )
+}
+
+void inf() : { }
+{
+    ( <INF> space() )
+}
+
+void neginf() : { }
+{
+    ( <NEGINF> space() )
+}
+
+void lbracket() : { }
+{
+    ( <LBRACKET> space() )
+}
+
+void rbracket() : { }
+{
+    ( <RBRACKET> space() )
+}
+
+void lbrace() : { }
+{
+    ( <LBRACE> space() )
+}
+
+void lcurly() : { }
+{
+    ( <LCURLY> space() )
+}
+
+void mod() : { }
+{
+    ( <INFIX_MOD> space() )
+}
+
+void mul() : { }
+{
+    ( <INFIX_MUL> space() )
+}
+
+void rbrace() : { }
+{
+    ( <RBRACE> space() )
+}
+
+void rcurly() : { }
+{
+    ( <RCURLY> space() )
+}
+
+void sub() : { }
+{
+    ( <INFIX_SUB> space() )
+}
+
+void scolon() : { }
+{
+    ( <SCOLON> space() )
+}
+
+void space() : { }
+{ 
+    ( <SPACE> )*
+}
diff --git a/container-search/src/main/javacc/com/yahoo/search/query/textserialize/parser/Parser.jj b/container-search/src/main/javacc/com/yahoo/search/query/textserialize/parser/Parser.jj
new file mode 100644
index 00000000000..52328ccafa5
--- /dev/null
+++ b/container-search/src/main/javacc/com/yahoo/search/query/textserialize/parser/Parser.jj
@@ -0,0 +1,205 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author <a href="mailto:tonyv@yahoo-inc.com">tonyv</a>
+ */
+options {
+    LOOKAHEAD = 1;
+    STATIC = false;
+    UNICODE_INPUT = true;
+}
+
+PARSER_BEGIN(Parser)
+package com.yahoo.search.query.textserialize.parser;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.LinkedHashMap;
+
+public class Parser {
+    private DispatchFormHandler dispatchFormHandler;
+    private Object dispatchContext;
+
+    public Parser(java.io.Reader stream, DispatchFormHandler dispatchFormHandler, Object dispatchContext) {
+        this(stream);
+        this.dispatchFormHandler = dispatchFormHandler;
+        this.dispatchContext = dispatchContext;
+    }
+
+    private static String parseString(String in) {
+        return stripEnclosingQuotes(
+            in.replace("\\\\", "\\").replace("\\\"", "\""));
+    }
+
+    private static String stripEnclosingQuotes(String in) {
+        return in.substring(1, in.length() - 1);
+    }
+}
+
+PARSER_END(Parser)
+
+Object start():
+{
+    final Object result;
+}
+{
+    result = form()
+    <EOF>
+
+    { return result; }
+}
+
+Object form():
+{
+    final Object result;
+}
+{
+    (
+    result = dispatchForm() |
+    result = map() |
+    result = array() |
+    result = literal()
+    )
+
+    { return result; }
+}
+
+Object literal():
+{
+    final Object result;
+}
+{
+    (
+    result = string() |
+    result = number() |
+    result = bool() |
+    result = null_()
+    )
+
+    { return result; }
+}
+
+String string():
+{
+    Token t;
+}
+{
+    t = <STRING>
+    { return parseString(t.image); }
+}
+
+Double number():
+{
+    Token t;
+}
+{
+    t = <NUMBER>
+    { return Double.valueOf(t.image); }
+}
+
+Boolean bool():
+{
+    Token t;
+}
+{
+    t = <BOOLEAN>
+
+    {
+     if ("on".equals(t.image)) return true;
+     return Boolean.valueOf(t.image);
+    }
+}
+
+Object null_():
+{}
+{
+    <NULL>
+    { return null; }
+}
+
+Map<Object, Object> map():
+{
+    Object key, value;
+    Map<Object, Object> result = new LinkedHashMap<Object, Object>();
+}
+{
+    <OPEN_CURLY>
+    (
+       key = form()
+       value = form()
+
+       { result.put(key, value); }
+    )*
+    <CLOSE_CURLY>
+
+    { return result; }
+}
+
+Object dispatchForm():
+{
+    Token symbol;
+    List<Object> arguments;
+}
+{
+    <OPEN_PAREN>
+    symbol = <SYMBOL>
+    arguments = forms()
+    <CLOSE_PAREN>
+
+    { return dispatchFormHandler.dispatch(symbol.image, arguments, dispatchContext); }
+}
+
+List<Object> forms():
+{
+    Object value;
+    List<Object> results = new ArrayList<Object>();
+}
+{
+    (
+        value = form()
+        { results.add(value); }
+    )*
+
+    { return results; }
+}
+
+List<Object> array():
+{
+    List<Object> result;
+}
+{
+    <OPEN_SQUARE>
+    result = forms()
+    <CLOSE_SQUARE>
+
+    { return result; }
+}
+
+SKIP : { " " | "\n" | "\r" | "\t" | "\f" | "," }
+
+TOKEN : { < OPEN_PAREN: "(" > }
+TOKEN : { < CLOSE_PAREN: ")" > }
+TOKEN : { < OPEN_CURLY: "{" > }
+TOKEN : { < CLOSE_CURLY: "}" > }
+TOKEN : { < OPEN_SQUARE: "[" > }
+TOKEN : { < CLOSE_SQUARE: "]" > }
+
+TOKEN : { <BOOLEAN: "true" | "false" | "on" | "off" > }
+TOKEN : { <NULL: "null" > }
+
+TOKEN : { < STRING: <QUOTE> (<STRING_CHARACTER>)* <QUOTE> > }
+TOKEN : { < #STRING_CHARACTER: "\\\\" | "\\\"" | ~["\"", "\\"] > }
+TOKEN : { < #QUOTE: "\"" > }
+
+TOKEN : { < NUMBER: (<SIGN>)? (<DIGIT>)+ ("." (<DIGIT>)*)? (<EXPONENT>)? > }
+TOKEN : { < #EXPONENT: ["e", "E"] (<SIGN>)? (<DIGIT>)+ > }
+TOKEN : { < #SIGN: ["+", "-"] >}
+TOKEN : { < #DIGIT: ["0" - "9"] > }
+
+TOKEN : { < SYMBOL: <SYMBOL_FIRST> (<SYMBOL_REST>)* > }
+TOKEN : { < #SYMBOL_FIRST: ["a"-"z"] | ["A"-"Z"] |
+        "+" | "-" | "*" | "/" |
+        "!" | "?" |
+        "-" | "_" |
+        "=" | ">" | "<"  > }
+TOKEN : { < #SYMBOL_REST: <SYMBOL_FIRST> | ["0" - "9"] | "." > }