add almost-exact copy of SDParser.jj

1 files changed, 2823 insertions, 0 deletions

diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj
new file mode 100644
index 00000000000..5a47ace5a1e
--- /dev/null
+++ b/config-model/src/main/javacc/IntermediateParser.jj
@@ -0,0 +1,2823 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+// Schema parser.
+//
+// NOTE: When this is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf
+
+options {
+    UNICODE_INPUT = true;
+    CACHE_TOKENS  = false;
+    STATIC = false;
+    DEBUG_PARSER = false;
+    ERROR_REPORTING = true;
+    FORCE_LA_CHECK = true;
+    USER_CHAR_STREAM = true;
+}
+
+PARSER_BEGIN(IntermediateParser)
+
+package com.yahoo.searchdefinition.parser;
+
+import com.yahoo.document.*;
+import com.yahoo.documentmodel.*;
+import com.yahoo.compress.Compressor;
+import com.yahoo.compress.CompressionType;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.searchdefinition.DistributableResource;
+import com.yahoo.searchdefinition.document.*;
+import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
+import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType;
+import com.yahoo.searchdefinition.RankingConstant;
+import com.yahoo.searchdefinition.OnnxModel;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.DocumentsOnlyRankProfile;
+import com.yahoo.searchdefinition.DefaultRankProfile;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings;
+import com.yahoo.searchdefinition.RankProfile.DiversitySettings;
+import com.yahoo.searchdefinition.Schema;
+import com.yahoo.searchdefinition.DocumentOnlySchema;
+import com.yahoo.searchdefinition.UnrankedRankProfile;
+import com.yahoo.searchdefinition.fieldoperation.*;
+import com.yahoo.searchlib.rankingexpression.FeatureList;
+import com.yahoo.searchlib.rankingexpression.evaluation.Value;
+import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorType;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.application.api.FileRegistry;
+import com.yahoo.config.model.api.ModelContext;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.Embedder;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.search.query.ranking.Diversity;
+import java.util.Optional;
+import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.logging.Level;
+
+/**
+ * The schema parser
+ *
+ * @author bratseth
+ */
+@SuppressWarnings("deprecation")
+public class IntermediateParser {
+
+    private DocumentTypeManager docMan = null;
+    private ApplicationPackage applicationPackage;
+    private FileRegistry fileRegistry;
+    private DeployLogger deployLogger;
+    private ModelContext.Properties properties;
+    private RankProfileRegistry rankProfileRegistry;
+    private boolean documentsOnly;
+
+    /**
+     * Creates a parser
+     *
+     * @param documentsOnly true to only parse the document aspect of a schema (e.g skip rank profiles)
+     */
+    public IntermediateParser(SimpleCharStream stream,
+                    ApplicationPackage applicationPackage,
+                    FileRegistry fileRegistry,
+                    DeployLogger deployLogger,
+                    ModelContext.Properties properties,
+                    RankProfileRegistry rankProfileRegistry,
+                    boolean documentsOnly) {
+        this(stream);
+        this.applicationPackage = applicationPackage;
+        this.fileRegistry = fileRegistry;
+        this.deployLogger = deployLogger;
+        this.properties = properties;
+        this.rankProfileRegistry = rankProfileRegistry;
+        this.documentsOnly = documentsOnly;
+    }
+
+    /**
+     * Consumes an indexing language script which will use the simple linguistics implementation
+     * for testing, by taking input from the current input stream.
+     *
+     * @param multiline Whether or not to allow multi-line expressions.
+     */
+    @SuppressWarnings("deprecation")
+    private IndexingOperation newIndexingOperation(boolean multiline) throws ParseException {
+        return newIndexingOperation(multiline, new SimpleLinguistics(), Embedder.throwsOnUse);
+    }
+
+    /**
+     * Consumes an indexing language script from the current input stream.
+     *
+     * @param multiline Whether or not to allow multi-line expressions.
+     * @param linguistics What to use for tokenizing.
+     */
+    private IndexingOperation newIndexingOperation(boolean multiline, Linguistics linguistics, Embedder embedder) throws ParseException {
+        SimpleCharStream input = (SimpleCharStream)token_source.input_stream;
+        if (token.next != null) {
+            input.backup(token.next.image.length());
+        }
+        try {
+            return IndexingOperation.fromStream(input, multiline, linguistics, embedder);
+        } finally {
+            token.next = null;
+            jj_ntk = -1;
+        }
+    }
+
+    /**
+     * Parses the given token image as a ranking expression feature list.
+     *
+     * @param image The token image to parse.
+     * @return The consumed feature list.
+     * @throws ParseException Thrown if the image could not be parsed.
+     */
+    private FeatureList getFeatureList(String image) throws ParseException {
+        try {
+            return new FeatureList(image);
+        }
+        catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) {
+            throw (ParseException) new ParseException("Could not parse feature list '" + image + "' at line " +
+                                                      token_source.input_stream.getBeginLine() + ", column " +
+                                                      token_source.input_stream.getBeginColumn() + ".").initCause(e);
+        }
+    }
+}
+
+PARSER_END(IntermediateParser)
+
+
+// --------------------------------------------------------------------------------
+//
+// Token declarations.
+//
+// --------------------------------------------------------------------------------
+
+// Declare white space characters. These do not include newline because it has
+// special meaning in several of the production rules.
+SKIP :
+{
+  " " | "\t" | "\r" | "\f"
+}
+
+// Declare all tokens to be recognized. When a word token is added it MUST be
+// added to the identifier() production rule.
+TOKEN :
+{
+  < NL: "\n" >
+| < ANNOTATION: "annotation" >
+| < ANNOTATIONREFERENCE: "annotationreference" >
+| < SCHEMA: "schema" >
+| < SEARCH: "search" >
+| < DIVERSITY: "diversity" >
+| < MIN_GROUPS: "min-groups" >
+| < CUTOFF_FACTOR: "cutoff-factor" >
+| < CUTOFF_STRATEGY: "cutoff-strategy" >
+| < LOOSE: "loose" >
+| < STRICT: "strict" >
+| < DOCUMENT: "document" >
+| < OPERATION: "operation" >
+| < ON_MATCH: "on-match" >
+| < ON_FIRST_PHASE: "on-first-phase" >
+| < ON_SECOND_PHASE: "on-second-phase" >
+| < ON_SUMMARY: "on-summary" >
+| < STRUCT: "struct" >
+| < INHERITS: "inherits" >
+| < FIELD: "field" >
+| < FIELDS: "fields" >
+| < FIELDSET: "fieldset" >
+| < STRUCTFIELD: "struct-field" >
+| < IMPORT: "import" >
+| < AS: "as" >
+| < INDEXING: "indexing" >
+| < SUMMARYTO: "summary-to" >
+| < DOCUMENTSUMMARY: "document-summary" >
+| < RANKTYPE: "rank-type" >
+| < WEIGHT: "weight" >
+| < TYPE: "type" >
+| < INDEX: "index" >
+| < MTOKEN: "token" >
+| < TEXT: "text" >
+| < WORD: "word" >
+| < GRAM: "gram" >
+| < GRAMSIZE: "gram-size" >
+| < MAXLENGTH: "max-length" >
+| < PREFIX: "prefix" >
+| < SUBSTRING: "substring" >
+| < SUFFIX: "suffix" >
+| < CONSTANT: "constant">
+| < ONNXMODEL: "onnx-model">
+| < MODEL: "model" >
+| < MUTATE: "mutate" >
+| < RANKPROFILE: "rank-profile" >
+| < RANKDEGRADATIONFREQ: "rank-degradation-frequency" >
+| < RANKDEGRADATION: "rank-degradation" >
+| < RAW_AS_BASE64_IN_SUMMARY: "raw-as-base64-in-summary" >
+| < RPBINSIZE: "doc-frequency" >
+| < RPBINLOW:  "min-fullrank-docs">
+| < RPPOSBINSIZE: "occurrences-per-doc" >
+| < SUMMARY: "summary" >
+| < FULL: "full" >
+| < STATIC: "static" >
+| < DYNAMIC: "dynamic" >
+| < MATCHEDELEMENTSONLY: "matched-elements-only" >
+| < SSCONTEXTUAL: "contextual" >
+| < SSOVERRIDE: "override" >
+| < SSTITLE: "title" >
+| < SSURL: "url" >
+| < PROPERTIES: "properties" >
+| < ATTRIBUTE: "attribute" >
+| < SORTING: "sorting" >
+| < DICTIONARY: "dictionary" >
+| < ASCENDING: "ascending" >
+| < DESCENDING: "descending" >
+| < UCA: "uca" >
+| < RAW: "raw" >
+| < LOWERCASE: "lowercase" >
+| < FUNCTION: "function" >
+| < LOCALE: "locale" >
+| < STRENGTH: "strength" >
+| < PRIMARY: "primary" >
+| < SECONDARY: "secondary" >
+| < TERTIARY: "tertiary" >
+| < QUATERNARY: "quaternary" >
+| < IDENTICAL: "identical" >
+| < STEMMING: "stemming" >
+| < NORMALIZING: "normalizing" >
+| < HASH: "hash" >
+| < BTREE: "btree" >
+| < CASED: "cased" >
+| < UNCASED: "uncased" >
+| < BOLDING: "bolding" >
+| < BODY: "body" >
+| < HEADER: "header" >
+| < NONE: "none" >
+| < ON: "on" >
+| < OFF: "off" >
+| < TRUE: "true" >
+| < FALSE: "false" >
+| < SYMMETRIC: "symmetric" >
+| < QUERYCOMMAND: "query-command" >
+| < ALIAS: "alias" >
+| < MATCH: "match" >
+| < RANK: "rank" >
+| < LITERAL: "literal" >
+| < EXACT: "exact" >
+| < FILTER: "filter" >
+| < NORMAL: "normal" >
+| < EXACTTERMINATOR: "exact-terminator" >
+| < INDEXINGREWRITE: "indexing-rewrite" >
+| < IGNOREDEFAULTRANKFEATURES: "ignore-default-rank-features" >
+| < ID: "id" >
+| < SOURCE: "source" >
+| < TO: "to" >
+| < DIRECT: "direct" >
+| < FROMDISK: "from-disk" >
+| < OMITSUMMARYFEATURES: "omit-summary-features" >
+| < ALWAYS: "always" >
+| < ONDEMAND: "on-demand" >
+| < NEVER: "never" >
+| < ENABLEBITVECTORS: "enable-bit-vectors" >
+| < ENABLEONLYBITVECTOR: "enable-only-bit-vector" >
+| < FASTACCESS: "fast-access" >
+| < MUTABLE: "mutable" >
+| < PAGED: "paged" >
+| < FASTSEARCH: "fast-search" >
+| < HUGE: "huge" >
+| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])+ ")" >
+| < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? >
+| < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? >
+| < COMPRESSION: "compression" >
+| < COMPRESSIONLEVEL: "level" >
+| < COMPRESSIONTHRESHOLD: "threshold" >
+| < LZ4: "lz4" >
+| < USEDOCUMENT: "use-document" >
+| < LBRACE: "{" >
+| < RBRACE: "}" >
+| < COLON: ":" >
+| < DOT: "." >
+| < COMMA: "," >
+| < ARRAY: "array" >
+| < WEIGHTEDSET: "weightedset" >
+| < MAP: "map" >
+| < REFERENCE: "reference" >
+| < QUESTIONMARK: "?" >
+| < CREATEIFNONEXISTENT: "create-if-nonexistent" >
+| < REMOVEIFZERO: "remove-if-zero" >
+| < MATCHPHASE: "match-phase" >
+| < EVALUATION_POINT: "evaluation-point" >
+| < PRE_POST_FILTER_TIPPING_POINT: "pre-post-filter-tipping-point" >
+| < ORDER: "order" >
+| < MAXFILTERCOVERAGE: "max-filter-coverage" >
+| < MAXHITS: "max-hits" >
+| < FIRSTPHASE: "first-phase" >
+| < SECONDPHASE: "second-phase" >
+| < MACRO: "macro" >
+| < INLINE: "inline" >
+| < ARITY: "arity" >
+| < LOWERBOUND: "lower-bound" >
+| < UPPERBOUND: "upper-bound" >
+| < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" >
+| < ENABLE_BM25: "enable-bm25" >
+| < HNSW: "hnsw" >
+| < MAXLINKSPERNODE: "max-links-per-node" >
+| < DISTANCEMETRIC: "distance-metric" >
+| < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" >
+| < MULTITHREADEDINDEXING: "multi-threaded-indexing" >
+| < MATCHFEATURES_SL: "match-features" (" ")* ":" (~["}","\n"])* ("\n")? >
+| < MATCHFEATURES_ML: "match-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
+| < MATCHFEATURES_ML_INHERITS: "match-features inherits " (<IDENTIFIER>) (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
+| < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? >
+| < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
+| < SUMMARYFEATURES_ML_INHERITS: "summary-features inherits " (<IDENTIFIER>) (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
+| < RANKFEATURES_SL: "rank-features" (" ")* ":" (~["}","\n"])* ("\n")? >
+| < RANKFEATURES_ML: "rank-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
+| < EXPRESSION_SL: "expression" (" ")* ":" (("{"<BRACE_SL_LEVEL_1>)|<BRACE_SL_CONTENT>)* ("\n")? >
+| < EXPRESSION_ML: "expression" (<SEARCHLIB_SKIP>)? "{" (("{"<BRACE_ML_LEVEL_1>)|<BRACE_ML_CONTENT>)* "}" >
+| < #BRACE_SL_LEVEL_1: (("{"<BRACE_SL_LEVEL_2>)|<BRACE_SL_CONTENT>)* "}" >
+| < #BRACE_SL_LEVEL_2: (("{"<BRACE_SL_LEVEL_3>)|<BRACE_SL_CONTENT>)* "}" >
+| < #BRACE_SL_LEVEL_3: <BRACE_SL_CONTENT> "}" >
+| < #BRACE_SL_CONTENT: (~["{","}","\n"])* >
+| < #BRACE_ML_LEVEL_1: (("{"<BRACE_ML_LEVEL_2>)|<BRACE_ML_CONTENT>)* "}" >
+| < #BRACE_ML_LEVEL_2: (("{"<BRACE_ML_LEVEL_3>)|<BRACE_ML_CONTENT>)* "}" >
+| < #BRACE_ML_LEVEL_3: <BRACE_ML_CONTENT> "}" >
+| < #BRACE_ML_CONTENT: (~["{","}"])* >
+| < #SEARCHLIB_SKIP: ([" ","\f","\n","\r","\t"])+ >
+| < RANKPROPERTIES: "rank-properties" >
+| < RERANKCOUNT: "rerank-count" >
+| < NUMTHREADSPERSEARCH: "num-threads-per-search" >
+| < MINHITSPERTHREAD: "min-hits-per-thread" >
+| < NUMSEARCHPARTITIONS: "num-search-partitions" >
+| < TERMWISELIMIT: "termwise-limit" >
+| < KEEPRANKCOUNT: "keep-rank-count" >
+| < RANKSCOREDROPLIMIT: "rank-score-drop-limit" >
+| < CONSTANTS: "constants" >
+| < FILE: "file" >
+| < URI: "uri" >
+| < IDENTIFIER:           ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_"])* >
+| < IDENTIFIER_WITH_DASH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])* >
+| < QUOTEDSTRING: "\"" ( ~["\""] )* "\"" >
+| < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* >
+| < DOUBLE: ("-")? (["0"-"9"])+ "." (["0"-"9"])+ >
+| < INTEGER: ("-")? (["0"-"9"])+ >
+| < LONG: ("-")? (["0"-"9"])+"L" >
+| < STRING: (["a"-"z","A"-"Z","_","0"-"9","."])+ >
+| < FILE_PATH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-", "/", "."])+ >
+| < HTTP: ["h","H"] ["t","T"] ["t","T"] ["p","P"] (["s","S"])? >
+| < URI_PATH: <HTTP> <COLON> ("//")? (["a"-"z","A"-"Z","0"-"9","_","-", "/", ".",":"])+ >
+| < LESSTHAN: "<" >
+| < GREATERTHAN: ">" >
+| < VARIABLE: "$" <IDENTIFIER> >
+| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? >
+| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? >
+}
+
+// Declare a special skip token for comments.
+SPECIAL_TOKEN :
+{
+  <SINGLE_LINE_COMMENT: "#" (~["\n","\r"])* >
+}
+
+
+// --------------------------------------------------------------------------------
+//
+// Production rules.
+//
+// --------------------------------------------------------------------------------
+
+/**
+ * The rule consumes any schema and returns the corresponding object. This is the only production that should
+ * ever consume leading newlines.
+ *
+ * @return the schema object
+ */
+Schema schema(DocumentTypeManager docMan) :
+{
+    this.docMan = docMan;
+    Schema schema;
+}
+{
+    (<NL>)* (schema = rootSchema() | schema = rootDocument())
+    { return schema; }
+}
+
+/**
+ * This rule consumes a proper schema block. This and rootDocument() are the only rules that should ever consume
+ * trailing newline tokens.
+ *
+ * @return the schema definition object.
+ */
+Schema rootSchema() :
+{
+    String name;
+    String inherited = null;
+    Schema schema;
+}
+{
+    ( ( <SCHEMA> | <SEARCH> ) name = identifier() (<INHERITS> inherited = identifier() )? {
+        schema = new Schema(name, applicationPackage, Optional.ofNullable(inherited), fileRegistry, deployLogger, properties);
+        rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants()));
+        rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants()));}
+      lbrace() (rootSchemaItem(schema) (<NL>)*)* <RBRACE> (<NL>)* <EOF>)
+    { return schema; }
+}
+
+/**
+ * Consumes an element of a schema block. This and rootSearch() are the only rules that should ever consume
+ * trailing newline tokens.
+ *
+ * @param schema the schema object to modify.
+ */
+void rootSchemaItem(Schema schema) : { }
+{
+    ( document(schema)
+      | rawAsBase64(schema)
+      | documentSummary(schema)
+      | field(null, schema)
+      | index(schema, null)
+      | rankingConstant(schema)
+      | rankProfile(schema)
+      | searchStemming(schema)
+      | useDocument(schema)
+      | structOutside(schema)
+      | annotationOutside(schema)
+      | fieldSet(schema)
+      | importField(schema)
+      | onnxModel(schema) )
+}
+
+/**
+ * Consumes a schema definition that contains only documents to be used for inheritance, etc.
+ *
+ * @return the schema definition object.
+ */
+Schema rootDocument() :
+{
+    Schema schema = new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties);
+}
+{
+    ( (rootDocumentItem(schema) (<NL>)*)*<EOF> )
+    { return schema; }
+}
+
+/**
+ * Consumes a single item from within a root document node.
+ *
+ * @param schema the schema object to modify.
+ */
+void rootDocumentItem(Schema schema) : { }
+{
+    ( namedDocument(schema) )
+}
+
+/**
+ * Consumes a use-document statement. This currently does nothing.
+ *
+ * @param schema the schema object to modify.
+ */
+void useDocument(Schema schema) : { }
+{
+    <USEDOCUMENT> <COLON> identifier()
+}
+
+/**
+ * Consumes a document element. The name defaults to the schema's name, but may be set.
+ *
+ * @param schema the schema object to add content to.
+ */
+void document(Schema schema) :
+{
+    String name = schema.getName();
+    SDDocumentType document;
+}
+{
+    ( <DOCUMENT> (name = identifier())? (<NL>)* { document = new SDDocumentType(name, schema); }
+      [ inheritsDocument(document) (<NL>)* ]
+      <LBRACE> (<NL>)* (documentBody(document, schema) (<NL>)*)* <RBRACE> )
+    {
+        schema.addDocument(document);
+    }
+}
+
+/**
+ * Consumes a document element, explicitly named
+ *
+ * @param schema the schema object to add content to
+ */
+void namedDocument(Schema schema) :
+{
+    String name;
+    SDDocumentType document;
+}
+{
+    ( <DOCUMENT> name = identifier() (<NL>)* { document = new SDDocumentType(name, schema); }
+      [ inheritsDocument(document) (<NL>)* ]
+      <LBRACE> (<NL>)* (documentBody(document, schema) (<NL>)*)* <RBRACE> )
+    {
+        schema.addDocument(document);
+    }
+}
+
+/**
+ * Consumes a document body block
+ *
+ * @param document the document type to modify.
+ * @param schema   the schema object to add content to
+ */
+void documentBody(SDDocumentType document, Schema schema) :
+{
+}
+{
+    ( annotation(schema, document)
+      | compression(document)
+      | headercfg(document)
+      | bodycfg(document)
+      | structInside(document, schema)
+      | field(document, schema) )
+}
+
+void rawAsBase64(Schema schema) :
+{
+    boolean enabled = false;
+}
+{
+    <RAW_AS_BASE64_IN_SUMMARY>
+    {
+        enabled = true;
+    }
+    [ <COLON> ( <TRUE> | ( <FALSE> { enabled = false; } ) ) ]
+    {
+        schema.enableRawAsBase64(enabled);
+    }
+}
+
+/**
+ * Consumes a document head block.
+ *
+ * @param document The document type to modify.
+ */
+void headercfg(SDDocumentType document) : { }
+{
+    <HEADER> lbrace() [compression(document) (<NL>)*] <RBRACE>
+}
+
+/**
+ * Consumes a document body block.
+ *
+ * @param document The document type to modify.
+ */
+void bodycfg(SDDocumentType document) : { }
+{
+    <BODY> lbrace() [compression(document) (<NL>)*] <RBRACE>
+}
+
+/**
+ * Consumes a compression block. This can be set in both document header and -body block.
+ *
+ * @param document The document type to modify.
+ */
+void compression(SDDocumentType document) :
+{
+    deployLogger.logApplicationPackage(Level.WARNING, "'compression' for a document is deprecated and ignored");
+}
+{
+    <COMPRESSION> lbrace() ( compressionItem() (<NL>)*)* <RBRACE>  { }
+}
+
+/**
+ * Consumes the body of a compression block.
+ *
+ */
+void compressionItem() :
+{ }
+{
+    ( ( <TYPE> <COLON> <LZ4> )
+      | (<COMPRESSIONTHRESHOLD> <COLON> <INTEGER>)
+      | (<COMPRESSIONLEVEL> <COLON> <INTEGER>)
+    ) { }
+}
+
+/**
+ * Consumes a document inheritance statement.
+ *
+ * @param document The document type to modify.
+ */
+void inheritsDocument(SDDocumentType document) :
+{
+    String name;
+}
+{
+    <INHERITS> name = identifier() { document.inherit(new DataTypeName(name)); }
+    ( <COMMA>  name = identifier() { document.inherit(new DataTypeName(name)); } )*
+}
+
+/**
+ * Consumes a field block from within a document element.
+ *
+ * @param document the document type to modify
+ * @param schema   the schema object to add content to
+ */
+void field(SDDocumentType document, Schema schema) :
+{
+    String name;
+    SDField field;
+    DataType type;
+}
+{
+    <FIELD> name = identifier() <TYPE> type = dataType()
+    {
+        if (name != null && Schema.isReservedName(name.toLowerCase())) {
+            throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
+        }
+        field = new TemporarySDField(name, type, document);
+    }
+    lbrace() (fieldBody(field, schema, document) (<NL>)*)* <RBRACE>
+    {
+        if (document != null) {
+            document.addField(field);
+        } else {
+            schema.addExtraField(field);
+        }
+    }
+}
+
+void fieldSet(Schema schema) :
+{
+  String setName;
+  String field;
+  String queryCommand;
+  List queryCommands = new ArrayList();
+  FieldOperationContainer matchSetting;
+  List matchSettings = new ArrayList();
+}
+{
+  <FIELDSET> setName = identifier() lbrace()
+    ((
+      ( <FIELDS><COLON> field = identifier() { schema.fieldSets().addUserFieldSetItem(setName, field); }
+      ( <COMMA> field = identifier() { schema.fieldSets().addUserFieldSetItem(setName, field); } )* )
+    |
+      ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { queryCommands.add(queryCommand); }
+    |
+      ( matchSetting = match(new SDField(setName, DataType.STRING)) ) { matchSettings.add(matchSetting); }
+    )(<NL>)*)+
+  <RBRACE>
+  {
+     // Apply settings after parsing since all user field items must be set first
+
+     for (Object command : queryCommands)
+         schema.fieldSets().userFieldSets().get(setName).queryCommands().add((String)command);
+
+     for (Object setting : matchSettings) {
+         ((SDField)setting).applyOperations();
+         schema.fieldSets().userFieldSets().get(setName).setMatching(((SDField)setting).getMatching());
+     }
+  }
+}
+
+/**
+ * This rule consumes a annotation block from within either a document element or a schema element.
+
+ * @param schema the schema object to add content to
+ */
+void annotationOutside(Schema schema) :
+{
+    String name;
+    SDAnnotationType type;
+}
+{
+    <ANNOTATION> name = identifier()
+    {
+        type = new SDAnnotationType(name.trim());
+    }
+    [ inheritsAnnotation(type) (<NL>)* ]
+    lbrace() (type = annotationBody(schema, type)) <RBRACE>
+    {
+        if (schema.getDocument()==null) throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one.");
+        schema.addAnnotation(type);
+    }
+}
+
+/**
+ * Consumes a annotation block from within either a document element.
+ *
+ * @param document the document object to add content to
+ */
+void annotation(Schema schema, SDDocumentType document) :
+{
+    String name;
+    SDAnnotationType type;
+}
+{
+    <ANNOTATION> name = identifier()
+    {
+        type = new SDAnnotationType(name.trim());
+    }
+    [ inheritsAnnotation(type) (<NL>)* ]
+    lbrace() (type = annotationBody(schema, type)) <RBRACE>
+    {
+        document.addAnnotation(type);
+    }
+}
+
+
+/**
+ * Consumes a single element of an annotation body block.
+ *
+ * @param schema   the schema object to add content to
+ * @param type     the type being built
+ * @return a modified or new AnnotationType instance
+ */
+SDAnnotationType annotationBody(Schema schema, SDAnnotationType type) :
+{
+    SDDocumentType struct = new SDDocumentType("annotation." + type.getName(), schema);
+}
+{
+    (structFieldDefinition(struct) (<NL>)*)*
+    {
+        if (struct.getFieldCount() > 0) { // Must account for the temporary TemporarySDField.
+            type = new SDAnnotationType(type.getName(), struct, type.getInherits());
+            struct.setStruct(null);
+        }
+        return type;
+    }
+}
+
+void inheritsAnnotation(SDAnnotationType annotation) :
+{
+    String name;
+}
+{
+    <INHERITS> name = identifier() { annotation.inherit(name); }
+}
+
+
+/**
+ * This rule consumes a struct block from within a document element.
+ *
+ * @param schema the schema object to add content to
+ */
+void structInside(SDDocumentType document, Schema schema) :
+{
+    SDDocumentType struct;
+}
+{
+    (
+        struct = structDefinition(schema, document)
+    )
+    {
+        document.addType(struct);
+    }
+}
+
+/**
+ * This rule consumes a struct block from within a document element.
+ *
+ * @param schema the schema object to add content to
+ */
+void structOutside(Schema schema) :
+{
+    SDDocumentType struct;
+}
+{
+    (
+        struct = structDefinition(schema, schema.getDocument())
+    )
+    {
+        schema.addType(struct);
+    }
+}
+
+/**
+ * This rule consumes a struct block from within a document element.
+ *
+ * @param schema the schema object to add content to
+ */
+SDDocumentType structDefinition(Schema schema, SDDocumentType repo) :
+{
+    String name;
+    String inherited = null;
+    SDDocumentType struct;
+}
+{
+    ( <STRUCT> name = identifier() (<NL>)* { struct = new SDDocumentType(name, schema); }
+    [ inheritsDocument(struct) (<NL>)* ]
+    lbrace() (structFieldDefinition(struct) (<NL>)*)* <RBRACE> )
+    {
+        try {
+            docMan.getDataType(name);
+            throw new ParseException("Reserved name '" + name + "' can not be used to declare a struct.");
+        } catch (IllegalArgumentException e) {
+            // empty
+        }
+        if (repo==null) throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one.");
+        SDDocumentType sdtype = repo.getOwnedType(struct.getDocumentName());
+        DataType stype = sdtype != null
+                         ? sdtype.getStruct()
+                         : TemporaryStructuredDataType.create(struct.getName());
+        struct.setStruct(stype);
+        return struct;
+    }
+}
+
+/**
+ * This rule consumes a data type block from within a field element.
+ *
+ * @return the consumed data type
+ */
+DataType dataType() :
+{
+    String typeName = null;
+    boolean isArrayOldStyle = false;
+    DataType mapType = null;
+    DataType arrayType = null;
+    DataType wsetType = null;
+    TensorType tensorType;
+    TemporaryStructuredDataType referenceType;
+}
+{
+    (   LOOKAHEAD(<ARRAY> <LESSTHAN>)               ( <ARRAY> <LESSTHAN> arrayType = dataType() <GREATERTHAN> { return DataType.getArray(arrayType); } )
+      | LOOKAHEAD(<WEIGHTEDSET> <LESSTHAN>)         ( <WEIGHTEDSET> <LESSTHAN> wsetType = dataType() <GREATERTHAN> { return  DataType.getWeightedSet(wsetType); } )
+      | LOOKAHEAD(<MAP> <LESSTHAN>)                 ( mapType = mapDataType() { return mapType; } )
+      | LOOKAHEAD(<ANNOTATIONREFERENCE> <LESSTHAN>) ( mapType = annotationRefDataType() { return mapType; } )
+      | LOOKAHEAD(<TENSOR_TYPE>)                    ( tensorType = tensorType("Field type") { return DataType.getTensor(tensorType); } )
+      | LOOKAHEAD(<REFERENCE>)                      ( <REFERENCE> <LESSTHAN> referenceType = referenceType() <GREATERTHAN> { return ReferenceDataType.createWithInferredId(referenceType); } )
+      | ( typeName = identifier() ["[]" { isArrayOldStyle = true; }] )
+    )
+    {
+        DataType type = VespaDocumentType.INSTANCE.getDataType(typeName);
+
+        if (type == null) {
+            // we are basically creating TemporaryStructDataType instances for ANYTHING here!!
+            // we must do this and clean them up later.
+            type = TemporaryStructuredDataType.create(typeName);
+        }
+
+        if (isArrayOldStyle) {
+            deployLogger.logApplicationPackage(Level.WARNING, "Data type syntax '" + typeName + "[]' is deprecated, use 'array<" + typeName + ">' instead.");
+            type = DataType.getArray(type);
+        }
+        if ("tag".equalsIgnoreCase(typeName) && type instanceof WeightedSetDataType) ((WeightedSetDataType)type).setTag(true);
+        return type;
+    }
+}
+
+TemporaryStructuredDataType referenceType() :
+{
+    String documentName;
+}
+{
+    ( documentName = identifier() )
+    {
+        return TemporaryStructuredDataType.create(documentName);
+    }
+}
+
+DataType annotationRefDataType() :
+{
+    DataType dataType;
+    String targetName;
+}
+{
+    ( <ANNOTATIONREFERENCE> <LESSTHAN> targetName = identifier() <GREATERTHAN> )
+    {
+        return new TemporaryAnnotationReferenceDataType(targetName);
+    }
+}
+
+DataType mapDataType() :
+{
+    DataType keyType;
+    DataType valType;
+}
+{
+  ( <MAP> <LESSTHAN> keyType = dataType() <COMMA> valType = dataType() <GREATERTHAN> )
+  {
+    return DataType.getMap(keyType, valType);
+  }
+
+}
+
+/* Note: not currently used, remove when decided that map type will not support
+polymorphism */
+DataType wildCardType() :
+{
+}
+{
+(<QUESTIONMARK>) { return DataType.NONE; }
+}
+
+/**
+ * This rule consumes a field block of a struct body.
+ *
+ * @param struct The struct to modify.
+ */
+void structFieldDefinition(SDDocumentType struct) :
+{
+    String name;
+    SDField field;
+    DataType type;
+}
+{
+    <FIELD> name = identifier() <TYPE> type = dataType() {
+        if (name != null && Schema.isReservedName(name.toLowerCase())) {
+            throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
+        }
+        field = new TemporarySDField(name, type, struct);
+        struct.addField(field);
+    }
+    lbrace() (id(field,struct) (<NL>)*)? (match(field) (<NL>)*)* <RBRACE> {
+    }
+}
+
+/**
+ * This rule consumes a struct subfield from a document field body. This is not to be confused with a document
+ * struct's fields, but rather this is a subfield of a document field of type struct.
+ *
+ * @param field    the field to modify
+ * @param schema   the schema object to add content to
+ * @param document the document type to modify
+ */
+void structField(FieldOperationContainer field, Schema schema, SDDocumentType document) :
+{
+    String name;
+    SDField structField;
+}
+{
+    <STRUCTFIELD> name = identifier() {
+        if (name != null && Schema.isReservedName(name.toLowerCase())) {
+            throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
+        }
+        FieldOperationContainer structFieldOp = new StructFieldOperation(name);
+        field.addOperation((StructFieldOperation) structFieldOp);
+    }
+    lbrace() (structFieldBody(structFieldOp, schema, document) (<NL>)*)* <RBRACE>
+}
+
+
+/**
+ * This rule consumes a single element of a field body block.
+ *
+ * @param field    the field being built
+ * @param schema   the schema object to add content to
+ * @param document the owning document, or null if this is a search field
+ */
+void fieldBody(SDField field, Schema schema, SDDocumentType document) : { }
+{
+    ( alias(field) |
+      attribute(field) |
+      body(field) |
+      bolding(field) |
+      dictionary(field) |
+      fieldStemming(field) |
+      header(field) |
+      id(field, document) |
+      summaryInField(field) |
+      index(schema, field) |
+      indexing(field) |
+      indexingRewrite(field) |
+      match(field) |
+      normalizing(field) |
+      queryCommand(field) |
+      rank(field) |
+      rankType(field) |
+      sorting(field, field.getName()) |
+      structField(field, schema, document) |
+      summaryTo(field) |
+      weight(field) |
+      weightedset(field) )
+}
+
+/**
+ * This rule consumes a single element of a struct subfield body block.
+ * Only elements that are supported in streaming schema and indexed schema (with complex attributes) are allowed.
+ *
+ * @param field    the field being built
+ * @param schema   the schema object to add content to
+ * @param document the owning document, or null if this is a schema field
+ */
+void structFieldBody(FieldOperationContainer field, Schema schema, SDDocumentType document) : { }
+{
+    ( summaryInField(field) |
+      indexing(field) |
+      attribute(field) |
+      match(field) |
+      queryCommand(field) |
+      structField(field, schema, document) |
+      summaryTo(field) )
+}
+
+/**
+ * This rule consumes an indexing block of a field element.
+ *
+ * @param field The field to modify.
+ */
+void indexing(FieldOperationContainer field) : { }
+{
+    ( <INDEXING> ( (<COLON> indexingOperation(field, false)) | indexingOperation(field, true) ) )
+}
+
+/**
+ * This rule consumes an IL script block. This is expected to consume trailing newlines.
+ *
+ * @param field The field to modify.
+ */
+void indexingOperation(FieldOperationContainer field, boolean multiLine) : { }
+{
+    { field.addOperation(newIndexingOperation(multiLine)); }
+}
+
+/**
+ * This rule consumes a summary-to statement of a field element.
+ *
+ * @param field The field to modify.
+ */
+void summaryTo(FieldOperationContainer field) :
+{
+    SummaryToOperation op = new SummaryToOperation();
+    String destination;
+    String name = field.getName();
+}
+{
+    <SUMMARYTO> [name = identifier()] <COLON> destination = identifier()
+    {
+        op.setName(name);
+        op.addDestination(destination);
+    }
+    ( <COMMA> destination = identifier() {op.addDestination(destination); } )*
+    {
+        field.addOperation(op);
+    }
+}
+
+
+/**
+ * This rule consumes a weight statement of a field element.
+ *
+ * @param field The field to modify.
+ */
+void weight(FieldOperationContainer field) :
+{
+    int num;
+}
+{
+    <WEIGHT> <COLON> num = integer()
+    {
+        WeightOperation op = new WeightOperation();
+        op.setWeight(num);
+        field.addOperation(op);
+    }
+}
+
+/**
+ * This rule consumes a weighted set statement of a field element.
+ *
+ * @param field The field to modify.
+ */
+void weightedset(FieldOperationContainer field) :
+{
+    WeightedSetOperation op = new WeightedSetOperation();
+}
+{
+    <WEIGHTEDSET> ( (<COLON> weightedsetBody(op))
+                    | (lbrace() (weightedsetBody(op) (<NL>)*)* <RBRACE>) )
+    {
+        field.addOperation(op);
+    }
+}
+
+/**
+ * This rule consumes one body item of a weighted set block.
+ *
+ * @param field The field to modify.
+ */
+void weightedsetBody(WeightedSetOperation field) : { }
+{
+    ( <CREATEIFNONEXISTENT> { field.setCreateIfNonExistent(true); }
+      | <REMOVEIFZERO>      { field.setRemoveIfZero(true); } )
+}
+
+/**
+ * This rule consumes a rank-type statement of a field element.
+ *
+ * @param field The field to modify.
+ */
+void rankType(FieldOperationContainer field) :
+{
+    String typeName;
+    String indexName = null;
+}
+{
+    <RANKTYPE> [indexName = identifier()] <COLON> typeName = identifier()
+    {
+        RankTypeOperation op = new RankTypeOperation();
+        op.setType(RankType.fromString(typeName));
+        op.setIndexName(indexName);
+        field.addOperation(op);
+    }
+}
+
+/**
+ * This rule consumes an attribute statement of a field element.
+ *
+ * @param field The field to modify.
+ */
+void attribute(FieldOperationContainer field) :
+{
+    String name = field.getName();
+}
+{
+    <ATTRIBUTE> [name = identifier()]
+    {
+        AttributeOperation op = new AttributeOperation(name);
+    }
+         ( (<COLON> attributeSetting(field, op, name))
+           | (lbrace() (attributeSetting(field, op, name) (<NL>)*)* <RBRACE>) )
+    {
+        field.addOperation(op);
+    }
+}
+
+void sorting(FieldOperationContainer field, String name) :
+{
+    SortingOperation op = new SortingOperation(name);
+}
+{
+    <SORTING>
+         ( (<COLON> sortingSetting(op, name))
+           | (lbrace() (sortingSetting(op, name) (<NL>)*)* <RBRACE>) )
+    {
+        field.addOperation(op);
+    }
+}
+
+void sortingSetting(SortingOperation sorting, String attributeName) :
+{
+    String locale;
+}
+{
+    (
+        <ASCENDING> { sorting.setAscending(); }
+      | <DESCENDING> { sorting.setDescending(); }
+      | <FUNCTION> <COLON> (
+                               <UCA>       { sorting.setFunction(Sorting.Function.UCA); }
+                             | <RAW>       { sorting.setFunction(Sorting.Function.RAW); }
+                             | <LOWERCASE> { sorting.setFunction(Sorting.Function.LOWERCASE); }
+                           )
+      | <STRENGTH> <COLON> (
+                               <PRIMARY>    { sorting.setStrength(Sorting.Strength.PRIMARY); }
+                             | <SECONDARY>  { sorting.setStrength(Sorting.Strength.SECONDARY); }
+                             | <TERTIARY>   { sorting.setStrength(Sorting.Strength.TERTIARY); }
+                             | <QUATERNARY> { sorting.setStrength(Sorting.Strength.QUATERNARY); }
+                             | <IDENTICAL>  { sorting.setStrength(Sorting.Strength.IDENTICAL); }
+                           )
+      | <LOCALE> <COLON> locale = identifierWithDash() { sorting.setLocale(locale); }
+    )
+}
+
+/**
+ * This rule consumes a single attribute setting statement of an attribute element.
+ *
+ * @param field The field to modify.
+ * @param attributeName The name of the attribute to change.
+ */
+void attributeSetting(FieldOperationContainer field, AttributeOperation attribute, String attributeName) :
+{
+    String str;
+}
+{
+    (
+        <HUGE>                 { attribute.setHuge(true); }
+      | <FASTSEARCH>           { attribute.setFastSearch(true); }
+      | <FASTACCESS>           { attribute.setFastAccess(true); }
+      | <MUTABLE>              { attribute.setMutable(true); }
+      | <PAGED>                { attribute.setPaged(true); }
+      | <ENABLEBITVECTORS>     { attribute.setEnableBitVectors(true); }
+      | <ENABLEONLYBITVECTOR>  { attribute.setEnableOnlyBitVector(true); }
+      | sorting(field, attributeName)
+      | <ALIAS> { String alias; String aliasedName=attributeName; } [aliasedName = identifier()] <COLON> alias = identifierWithDash() {
+          attribute.setDoAlias(true);
+          attribute.setAlias(alias);
+          attribute.setAliasedName(aliasedName);
+      }
+      | attributeTensorType(attribute)
+      | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { attribute.setDistanceMetric(str); }
+    )
+}
+
+/**
+ * This rule consumes a tensor type statement for an attribute element.
+ *
+ * @param attribute The attribute to modify.
+ */
+void attributeTensorType(AttributeOperation attribute) :
+{
+    TensorType tensorType;
+}
+{
+    tensorType = tensorType("For attribute field '" + attribute.getName() + "'")
+    {
+        // TODO: Remove on Vespa 8
+        deployLogger.logApplicationPackage(Level.WARNING, "In field '" + attribute.getName() + "': Specifying tensor type on the attribute is deprecated and has no effect.");
+    }
+}
+
+/**
+ * This rule consumes a summary statement defined inside a document-summary block.
+ *
+ * @param document The document summary to modify.
+ */
+void summaryInDocument(DocumentSummary document) :
+{
+    String name;
+    DataType type;
+    SummaryField summary;
+
+}
+{
+    <SUMMARY> name = identifierWithDash() { }
+    <TYPE>    type = dataType()   {
+        summary = new SummaryField(name, type);
+        summary.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE);
+
+        SummaryInFieldLongOperation op = new SummaryInFieldLongOperation();
+    }
+    lbrace() (summaryItem(op) (<NL>)*)* <RBRACE>
+    {
+        if (op.destinationIterator().hasNext()) {
+            throw new ParseException("Summaries defined in a document-summary section " +
+                                     "can not have a 'to' line.");
+        }
+        op.applyToSummary(summary);
+        document.add(summary);
+    }
+}
+
+/**
+ * The rule consumes a summary statement defined inside a field.
+ *
+ * @param field The field to modify.
+ */
+void summaryInField(FieldOperationContainer field) :
+{
+    SummaryInFieldOperation summary;
+}
+{
+    ( <SUMMARY> ( LOOKAHEAD(2) summary = summaryInFieldShort(field)
+                  | summary = summaryInFieldLong(field)) )
+    {
+        field.addOperation(summary);
+    }
+}
+
+/**
+ * This rule consumes a single-line summary field.
+ *
+ * @param field The field to modify.
+ * @return The consumed summary field.
+ */
+SummaryInFieldOperation summaryInFieldShort(FieldOperationContainer field) :
+{
+    String name = field.getName();
+    SummaryField ret;
+}
+{
+    [ name = identifier() ]
+    {
+        SummaryInFieldShortOperation op = new SummaryInFieldShortOperation(name);
+    }
+    <COLON> ( <DYNAMIC> { op.setTransform(SummaryTransform.DYNAMICTEASER);
+                          op.addSource(name);
+                        }
+              | <MATCHEDELEMENTSONLY> { op.setTransform(SummaryTransform.MATCHED_ELEMENTS_FILTER); }
+              | (<FULL> | <STATIC>) { op.setTransform(SummaryTransform.NONE); } )
+    { return op; }
+}
+
+/**
+ * This rule consumes a multi-line summary field.
+ *
+ * @return The consumed summary field.
+ */
+SummaryInFieldOperation summaryInFieldLong(FieldOperationContainer field) :
+{
+    String name = field.getName();
+    DataType type = null;
+}
+{
+    ( [ name = identifier() [ <TYPE> type = dataType() ] ]
+      lbrace()
+      {
+          SummaryInFieldLongOperation op = new SummaryInFieldLongOperation(name);
+          op.setType(type);
+      }
+      (summaryItem(op) (<NL>)*)* <RBRACE> )
+    { return op; }
+}
+
+/**
+ * This rule consumes an item of a summary field block.
+ *
+ * @param field The field to modify.
+ */
+void summaryItem(SummaryInFieldLongOperation field) : { }
+{
+    ( summaryTransform(field)
+      | summaryBolding(field)
+      | summarySourceList(field)
+      | summaryDestinationList(field)
+      | summaryProperties(field) )
+}
+
+/**
+ * This rule consumes a transform statement for a summary field element.
+ *
+ * @param field            The field to modify.
+ */
+void summaryTransform(SummaryInFieldOperation field) : { }
+{
+    ( <DYNAMIC>             { field.setTransform(SummaryTransform.DYNAMICTEASER); }
+     | <MATCHEDELEMENTSONLY> { field.setTransform(SummaryTransform.MATCHED_ELEMENTS_FILTER); }
+     | (<FULL> | <STATIC>) { field.setTransform(SummaryTransform.NONE); } )
+}
+
+/**
+ * This rule consumes a bolding statement for a summary field element.
+ *
+ * @param field The summary field to modify.
+ */
+void summaryBolding(SummaryInFieldLongOperation field) :
+{
+    boolean bold;
+}
+{
+    <BOLDING> <COLON> bold = bool()
+    { field.setBold(bold); }
+}
+
+/**
+ * This rule consumes a source-list statement for a summary field element.
+ *
+ * @param field The summary field to modify.
+ */
+void summarySourceList(SummaryInFieldOperation field) :
+{
+    String str;
+}
+{
+    ( <SOURCE> <COLON> str = identifier() { field.addSource(str); }
+      (        <COMMA> str = identifier() { field.addSource(str); } )* ) +
+}
+
+/**
+ * This rule consumes a destination-list statement for a summary field element.
+ *
+ * @param field The summary field to modify.
+ */
+void summaryDestinationList(SummaryInFieldLongOperation field) :
+{
+    String str;
+}
+{
+    <TO> <COLON> str = identifier() { field.addDestination(str); }
+    (    <COMMA> str = identifier() { field.addDestination(str); } )*
+}
+
+/**
+ * This rule consumes properties for a summary field element.
+ *
+ * @param field The summary field to modify.
+ */
+void summaryProperties(SummaryInFieldLongOperation field) : { }
+{
+    <PROPERTIES> lbrace() (summaryProperty(field) <NL>)+ <RBRACE>
+}
+
+/**
+ * This rule consumes a single summary property pair for a summary field element.
+ *
+ * @param field The summary field to modify.
+ */
+void summaryProperty(SummaryInFieldLongOperation field) :
+{
+    String name, value;
+}
+{
+    name = identifierWithDash() <COLON> (value = identifierWithDash() | value = quotedString())
+    { field.addProperty(new SummaryField.Property(name, value)); }
+}
+
+/**
+ * This rule consumes a stemming block of a field element.
+ *
+ * @param field The field to modify.
+ */
+void fieldStemming(FieldOperationContainer field) :
+{
+    String setting;
+    StemmingOperation op = new StemmingOperation();
+}
+{
+    <STEMMING> <COLON> setting = identifierWithDash()
+    {
+        op.setSetting(setting);
+        field.addOperation(op);
+    }
+}
+
+/**
+ * This rule consumes a stemming statement for a schema element.
+ *
+ * @param schema the schema to modify
+ */
+void searchStemming(Schema schema) :
+{
+    String setting;
+}
+{
+    <STEMMING> <COLON> setting = identifierWithDash()
+    { schema.setStemming(Stemming.get(setting)); }
+}
+
+/**
+ * This rule consumes a normalizing statement of a field element. At the moment, this can only be used to turn off
+ * normalizing.
+ *
+ * @param field The field to modify.
+ */
+void normalizing(FieldOperationContainer field) :
+{
+    String setting;
+}
+{
+    <NORMALIZING> <COLON> setting = identifierWithDash()
+    {
+        field.addOperation(new NormalizingOperation(setting));
+    }
+}
+
+/**
+ * This rule consumes a bolding statement of a field element.
+ *
+ * @param field The field to modify.
+ */
+void bolding(FieldOperationContainer field) :
+{
+    boolean bold;
+}
+{
+    <BOLDING> <COLON> bold = bool()
+    {
+        field.addOperation(new BoldingOperation(bold));
+    }
+}
+
+/**
+ * This rule consumes a dictionary statement of a field element.
+ *
+ * @param field The field to modify.
+ */
+void dictionary(FieldOperationContainer field) :
+{
+}
+{
+    <DICTIONARY>
+    ( (<COLON> dictionarySetting(field))
+    | (lbrace() (dictionarySetting(field) (<NL>)*)* <RBRACE>))
+    {
+    }
+}
+
+void dictionarySetting(FieldOperationContainer field) :
+{
+    Dictionary.Type type;
+}
+{
+    (   <HASH>            { field.addOperation(new DictionaryOperation(DictionaryOperation.Operation.HASH)); }
+      | <BTREE>           { field.addOperation(new DictionaryOperation(DictionaryOperation.Operation.BTREE)); }
+      | <CASED>           { field.addOperation(new DictionaryOperation(DictionaryOperation.Operation.CASED)); }
+      | <UNCASED>           { field.addOperation(new DictionaryOperation(DictionaryOperation.Operation.UNCASED)); })
+    {
+    }
+}
+
+/**
+ * This rule consumes a body statement of a field element.
+ *
+ * @param field The field to modify.
+ */
+void body(SDField field) : { }
+{
+    <BODY>
+    {
+        deployLogger.logApplicationPackage(Level.WARNING, field + ": 'header/body' is deprecated and has no effect.");
+    }
+}
+
+/**
+ * This rule consumes a header statement of a field element.
+ *
+ * @param field The field to modify.
+ */
+void header(SDField field) : { }
+{
+    <HEADER>
+    {
+        deployLogger.logApplicationPackage(Level.WARNING, field + ": 'header/body' is deprecated and has no effect.");
+    }
+}
+
+void queryCommand(FieldOperationContainer container) :
+{
+    String command;
+    QueryCommandOperation field = new QueryCommandOperation();
+}
+{
+    <QUERYCOMMAND> <COLON> ( command = identifierWithDash() | command = quotedString() )
+    {
+        field.addQueryCommand(command);
+        container.addOperation(field);
+    }
+}
+
+void alias(FieldOperationContainer container) :
+{
+    String aliasedName = null;
+    String alias;
+}
+{
+    <ALIAS> [aliasedName = identifier()] <COLON> alias = identifierWithDash()
+    {
+       AliasOperation op = new AliasOperation(aliasedName, alias);
+       container.addOperation(op);
+    }
+}
+
+FieldOperationContainer match(FieldOperationContainer field) : { }
+{
+    <MATCH> ( (<COLON> matchType(field))
+              | (lbrace() (matchItem(field) (<NL>)*)* <RBRACE>) )
+    { return field; }
+}
+
+/**
+ * This rule consumes a single match item for a match block.
+ *
+ * @param field The field to modify.
+ */
+void matchItem(FieldOperationContainer field) : { }
+{
+    ( matchType(field) | exactTerminator(field) | gramSize(field) | matchSize(field) )
+}
+
+void matchType(FieldOperationContainer container) :
+{
+    MatchOperation matchOp = new MatchOperation();
+}
+{
+    (   <MTOKEN>    { matchOp.setMatchingType(Matching.Type.TEXT); } // Deprecated synonym to TEXT
+      | <TEXT>      { matchOp.setMatchingType(Matching.Type.TEXT); }
+      | <WORD>      { matchOp.setMatchingType(Matching.Type.WORD); }
+      | <EXACT>     { matchOp.setMatchingType(Matching.Type.EXACT); }
+      | <GRAM>      { matchOp.setMatchingType(Matching.Type.GRAM); }
+      | <CASED>     { matchOp.setCase(Case.CASED); }
+      | <UNCASED>   { matchOp.setCase(Case.UNCASED); }
+      | <PREFIX>    { matchOp.setMatchingAlgorithm(Matching.Algorithm.PREFIX); }
+      | <SUBSTRING> { matchOp.setMatchingAlgorithm(Matching.Algorithm.SUBSTRING); }
+      | <SUFFIX>    { matchOp.setMatchingAlgorithm(Matching.Algorithm.SUFFIX); } )
+    {
+        container.addOperation(matchOp);
+    }
+}
+
+void exactTerminator(FieldOperationContainer container) :
+{
+    String terminator;
+    MatchOperation field = new MatchOperation();
+}
+{
+    <EXACTTERMINATOR> <COLON> terminator = quotedString()
+    {
+        field.setExactMatchTerminator(terminator);
+        container.addOperation(field);
+    }
+}
+
+void gramSize(FieldOperationContainer container) :
+{
+    int gramSize;
+    MatchOperation field = new MatchOperation();
+}
+{
+    <GRAMSIZE> <COLON> gramSize = integer()
+    {
+        field.setGramSize(gramSize);
+        container.addOperation(field);
+    }
+}
+
+void matchSize(FieldOperationContainer container) :
+{
+    int matchSize;
+    MatchOperation field = new MatchOperation();
+}
+{
+    <MAXLENGTH> <COLON> matchSize = integer()
+    {
+        field.setMaxLength(matchSize);
+        container.addOperation(field);
+    }
+}
+/**
+ * Consumes a rank statement of a field element.
+ *
+ * @param field The field to modify.
+ */
+void rank(FieldOperationContainer field) :
+{
+    RankOperation op = new RankOperation();
+}
+{
+    <RANK> ( (<COLON> rankSetting(op))
+             | (lbrace() (rankSetting(op) (<NL>)*)* <RBRACE>) )
+    {
+        field.addOperation(op);
+    }
+}
+
+/**
+ * Consumes a single rank setting of a rank statement.
+ *
+ * @param field The field to modify.
+ */
+void rankSetting(RankOperation field) : { }
+{
+    ( <LITERAL>   { field.setLiteral(true); }
+      | <NORMAL>  { field.setNormal(true); }
+      | <FILTER>  { field.setFilter(true); } )
+}
+
+/**
+ * Consumes an id statement of a field body block.
+ *
+ * @param field    The field to modify.
+ * @param document The document type to modify.
+ */
+void id(FieldOperationContainer field, SDDocumentType document) :
+{
+    int fieldId;
+    IdOperation op = new IdOperation();
+}
+{
+    <ID> <COLON> fieldId = integer()
+    {
+        op.setDocument(document);
+        op.setFieldId(fieldId);
+        field.addOperation(op);
+    }
+}
+
+/**
+ * Consumes an indexing-rewrite statement of a field body block.
+ *
+ * @param field The field to modify.
+ */
+void indexingRewrite(FieldOperationContainer field) : { }
+{
+    <INDEXINGREWRITE> <COLON> <NONE>
+    { field.addOperation(new IndexingRewriteOperation()); }
+}
+
+/**
+ * Consumes a document-summary block from within a schema block.
+ *
+ * @param schema the schema object to add content to
+ */
+void documentSummary(Schema schema) :
+{
+    String name;
+    DocumentSummary summary;
+}
+{
+    ( <DOCUMENTSUMMARY>
+      name = identifierWithDash() { schema.addSummary(summary = new DocumentSummary(name, schema)); }
+      [inheritsDocumentSummary(summary, schema)]
+      lbrace()
+         (
+           <FROMDISK> { summary.setFromDisk(true); } |
+           <OMITSUMMARYFEATURES> { summary.setOmitSummaryFeatures(true); } |
+           documentSummaryItem(summary) |
+           <NL>
+         )*
+      <RBRACE>
+      )
+}
+
+/**
+ * This rule consumes an inherits statement of a document summary.
+ *
+ * @param documentSummary the document summary to modify
+ * @param schema the schema object documentSummary is being added to
+ */
+void inheritsDocumentSummary(DocumentSummary documentSummary, Schema schema) :
+{
+    String name;
+}
+{
+    <INHERITS> name = identifierWithDash()
+    {
+        documentSummary.setInherited(name);
+    }
+}
+
+/**
+ * Consumes a single document-summary item.
+ *
+ * @param summary The document summary to modify.
+ */
+void documentSummaryItem(DocumentSummary summary) : { }
+{
+    summaryInDocument(summary)
+}
+
+/**
+ * Consumes an index block for a field element.
+ *
+ * @param schema the schema object to add content to
+ * @param field  the field to modify
+ */
+void index(Schema schema, FieldOperationContainer field) :
+{
+    IndexOperation op = new IndexOperation();
+    String indexName = (field != null) ? field.getName() : null;
+}
+{
+    <INDEX> [indexName = identifier()]
+    {
+        if (indexName == null) {
+            throw new ParseException("Index statements outside fields must have an explicit name.");
+        }
+        op.setIndexName(indexName);
+    }
+    ( (<COLON> indexBody(op) (<COMMA> indexBody(op))*) |
+      (lbrace() (indexBody(op) (<NL>)*)* <RBRACE>) )
+    {
+        if (field == null) {
+
+            Index index = new Index(indexName);
+            op.applyToIndex(index);
+            schema.addIndex(index);
+        } else {
+            field.addOperation(op);
+        }
+    }
+}
+
+/**
+ * Consumes a single index statement for an index block.
+ *
+ * @param index The index to modify.
+ */
+void indexBody(IndexOperation index) :
+{
+    String str;
+    int arity;
+    long num;
+    double threshold;
+}
+{
+    ( <PREFIX>                                           { index.setPrefix(true); }
+      | <ALIAS> <COLON> str = identifierWithDash()       { index.addAlias(str); }
+      | <STEMMING> <COLON> str = identifierWithDash()    { index.setStemming(str); }
+      | <ARITY> <COLON> arity = integer()                              { index.setArity(arity); }
+      | <LOWERBOUND> <COLON> num = consumeLong()                       { index.setLowerBound(num); }
+      | <UPPERBOUND> <COLON> num = consumeLong()                       { index.setUpperBound(num); }
+      | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); }
+      | <ENABLE_BM25>                                                  { index.setEnableBm25(true); }
+      | hnswIndex(index)                                               { }
+    )
+}
+
+void hnswIndex(IndexOperation index) :
+{
+    HnswIndexParams.Builder params = new HnswIndexParams.Builder();
+}
+{
+    ( LOOKAHEAD(<HNSW> lbrace())
+      <HNSW> ( (lbrace() (hnswIndexBody(params) (<NL>)*)* <RBRACE>) ) |
+      <HNSW> )
+    {
+        index.setHnswIndexParams(params);
+    }
+}
+
+void hnswIndexBody(HnswIndexParams.Builder params) :
+{
+    int num;
+    boolean bool;
+}
+{
+    ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); }
+      | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); }
+      | <MULTITHREADEDINDEXING> <COLON> bool = bool() { params.setMultiThreadedIndexing(bool); } )
+}
+
+/**
+ * Consumes a onnx-model block of a schema element.
+ *
+ * @param schema the schema object to add content to.
+ */
+void onnxModel(Schema schema) :
+{
+    String name;
+    OnnxModel onnxModel;
+}
+{
+    ( <ONNXMODEL> name = identifier()
+        {
+            onnxModel = new OnnxModel(name);
+        }
+      lbrace() (onnxModelItem(onnxModel) (<NL>)*)+ <RBRACE> )
+    {
+        if (documentsOnly) return;
+        schema.onnxModels().add(onnxModel);
+    }
+}
+
+/**
+ * This rule consumes an onnx-model block.
+ *
+ * @param onnxModel The onnxModel to modify.
+ */
+void onnxModelItem(OnnxModel onnxModel) :
+{
+    String path = null;
+}
+{
+    (
+        (path = fileItem()) { onnxModel.setFileName(path); } |
+        (path = uriItem()) { onnxModel.setUri(path); } |
+        (<ONNX_INPUT_SL>) {
+            String name = token.image.substring(5, token.image.lastIndexOf(":")).trim();
+            if (name.startsWith("\"")) { name = name.substring(1, name.length() - 1); }
+            String source = token.image.substring(token.image.lastIndexOf(":") + 1).trim();
+            onnxModel.addInputNameMapping(name, source);
+        } |
+        (<ONNX_OUTPUT_SL>) {
+            String name = token.image.substring(6, token.image.lastIndexOf(":")).trim();
+            if (name.startsWith("\"")) { name = name.substring(1, name.length() - 1); }
+            String as = token.image.substring(token.image.lastIndexOf(":") + 1).trim();
+            onnxModel.addOutputNameMapping(name, as);
+        }
+    )
+}
+
+/**
+ * Consumes a constant block of a schema element.
+ *
+ * @param schema the schema object to add content to
+ */
+void rankingConstant(Schema schema) :
+{
+    String name;
+    String path = null;
+    DistributableResource.PathType pathType = DistributableResource.PathType.FILE;
+    TensorType type = null;
+}
+{
+    ( <CONSTANT> name = identifier() lbrace()
+      (path = fileItem() { pathType = DistributableResource.PathType.FILE; }
+       | path = uriItem() { pathType = DistributableResource.PathType.URI; }
+       | type = tensorTypeWithPrefix(rankingConstantErrorMessage(name)) (<NL>)*
+      )+
+      <RBRACE>
+    )
+    {
+        if (documentsOnly) return;
+        schema.rankingConstants().add(new RankingConstant(name, type, path, pathType));
+    }
+}
+
+String fileItem() :
+{
+   String path;
+}
+{
+  (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = token.image; } { } (<NL>)*) { return path; }
+}
+String uriItem() :
+{
+   String path;
+}
+{
+  (<URI> <COLON> ( <URI_PATH> ) { path = token.image; } (<NL>)*) { return path; }
+}
+
+String rankingConstantErrorMessage(String name) : {}
+{
+    { return "For ranking constant ' " + name + "'"; }
+}
+
+/**
+ * Consumes a rank-profile block of a schema element.
+ *
+ * @param schema the schema object to add content to
+ */
+void rankProfile(Schema schema) :
+{
+    String name;
+    RankProfile profile;
+}
+{
+    ( ( <MODEL> | <RANKPROFILE> ) name = identifierWithDash()
+        {
+            if (documentsOnly) {
+                profile = new DocumentsOnlyRankProfile(name, schema, rankProfileRegistry, schema.rankingConstants());
+            }
+            else if ("default".equals(name)) {
+                profile = rankProfileRegistry.get(schema, "default");
+            } else {
+                profile = new RankProfile(name, schema, rankProfileRegistry, schema.rankingConstants());
+            }
+        }
+      [inheritsRankProfile(profile)]
+      lbrace() (rankProfileItem(profile) (<NL>)*)* <RBRACE> )
+    {
+        if (documentsOnly) return;
+        rankProfileRegistry.add(profile);
+    }
+}
+
+/**
+ * This rule consumes a single statement for a rank-profile block.
+ *
+ * @param profile The rank profile to modify.
+ */
+void rankProfileItem(RankProfile profile) : { }
+{
+    ( fieldRankType(profile)
+      | fieldWeight(profile)
+      | fieldRankFilter(profile)
+      | firstPhase(profile)
+      | matchPhase(profile)
+      | function(profile)
+      | mutate(profile)
+      | ignoreRankFeatures(profile)
+      | numThreadsPerSearch(profile)
+      | minHitsPerThread(profile)
+      | numSearchPartitions(profile)
+      | termwiseLimit(profile)
+      | rankFeatures(profile)
+      | rankProperties(profile)
+      | secondPhase(profile)
+      | rankDegradation(profile)
+      | constants(profile)
+      | matchFeatures(profile)
+      | summaryFeatures(profile) )
+}
+
+/**
+ * This rule consumes an inherits statement of a rank-profile.
+ *
+ * @param profile the profile to modify
+ */
+void inheritsRankProfile(RankProfile profile) :
+{
+    String name;
+}
+{
+    <INHERITS> name = identifierWithDash() { profile.inherit(name); }
+    ( <COMMA>  name = identifierWithDash() { profile.inherit(name);; } )*
+}
+
+/**
+ * This rule consumes an mutate statement of a rank-profile.
+ *
+ * @param profile The profile to modify.
+ */
+void mutate(RankProfile profile) :
+{
+}
+{
+    <MUTATE> lbrace() (mutate_operation(profile) <NL>)+ <RBRACE>
+    {  }
+}
+
+void mutate_operation(RankProfile profile) :
+{
+    String attribute, operation;
+    RankProfile.MutateOperation.Phase phase;
+}
+{
+   ( <ON_MATCH> { phase = RankProfile.MutateOperation.Phase.on_match; }
+   | <ON_FIRST_PHASE> { phase = RankProfile.MutateOperation.Phase.on_first_phase; }
+   | <ON_SECOND_PHASE> { phase = RankProfile.MutateOperation.Phase.on_second_phase; }
+   | <ON_SUMMARY> { phase = RankProfile.MutateOperation.Phase.on_summary; }
+   )
+   lbrace() attribute = identifier() operation = mutate_expr() (<NL>)* <RBRACE>
+   { profile.addMutateOperation(phase, attribute, operation); }
+}
+
+String mutate_expr() :
+{
+     String op;
+     Number constant = null;
+}
+{
+    (("+=" | "-=" | "=") { op = token.image; } constant = consumeNumber())
+    { return constant != null ? (op + constant) : op; }
+}
+
+/**
+ * This rule consumes a function statement of a rank-profile.
+ *
+ * @param profile The profile to modify.
+ */
+void function(RankProfile profile) :
+{
+    String name, expression, parameter;
+    List parameters = new ArrayList();
+    boolean inline = false;
+}
+{
+    (  ( <FUNCTION> | <MACRO> ) inline = inline() name = identifier() [ "$" { name = name + token.image; } ]
+      "("
+          [ parameter = identifier()         { parameters.add(parameter); }
+          ( <COMMA> parameter = identifier() { parameters.add(parameter); } )* ]
+      ")"
+      lbrace() expression = expression() (<NL>)* <RBRACE> )
+    { profile.addFunction(name, parameters, expression, inline); }
+}
+
+boolean inline() :
+{
+}
+{
+    ( <INLINE> { return true; } ) ?
+    { return false; }
+}
+
+/**
+ * This rule consumes a match-phase block of a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void matchPhase(RankProfile profile) :
+{
+    MatchPhaseSettings settings = new MatchPhaseSettings();
+}
+{
+    <MATCHPHASE> lbrace() (matchPhaseItem(settings) (<NL>)*)* <RBRACE>
+    {
+        settings.checkValid();
+ 	    profile.setMatchPhaseSettings(settings);
+    }
+}
+
+void matchPhaseItem(MatchPhaseSettings settings) :
+{
+    String str;
+    int num;
+    double multiplier;
+    double coverage;
+}
+{
+    ( <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); }
+      | diversity(settings)
+      | <ORDER> <COLON> ( <ASCENDING> { settings.setAscending(true); } 
+                          | <DESCENDING> { settings.setAscending(false); } )
+      | <MAXHITS> <COLON> num = integer() { settings.setMaxHits(num); }
+      | <MAXFILTERCOVERAGE> <COLON> coverage = consumeFloat() { settings.setMaxFilterCoverage(coverage); }
+      | <EVALUATION_POINT> <COLON> multiplier = consumeFloat() { settings.setEvaluationPoint(multiplier); }
+      | <PRE_POST_FILTER_TIPPING_POINT> <COLON> multiplier = consumeFloat() { settings.setPrePostFilterTippingPoint(multiplier); }
+    )
+    { return; }
+}
+
+/**
+ * This rule consumes a diversity block of a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void diversity(MatchPhaseSettings profile) :
+{
+    DiversitySettings settings = new DiversitySettings();
+}
+{
+    <DIVERSITY> lbrace() (diversityItem(settings) (<NL>)*)* <RBRACE>
+    {
+ 	    profile.setDiversity(settings);
+    }
+}
+
+void diversityItem(DiversitySettings settings) :
+{
+    String str;
+    int num;
+    double multiplier;
+}
+{
+    (   <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); }
+      | <MIN_GROUPS> <COLON> num = integer() { settings.setMinGroups(num); }
+      | <CUTOFF_FACTOR> <COLON> multiplier = consumeFloat() { settings.setCutoffFactor(multiplier); }
+      | <CUTOFF_STRATEGY> <COLON>
+        (   <STRICT> { settings.setCutoffStrategy(Diversity.CutoffStrategy.strict); }
+          | <LOOSE>  { settings.setCutoffStrategy(Diversity.CutoffStrategy.loose); }
+        )
+    )
+    { return; }
+}
+
+
+
+/**
+ * Consumes the first-phase block of a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void firstPhase(RankProfile profile) :
+{
+    String exp;
+}
+{
+    <FIRSTPHASE> lbrace() (firstPhaseItem(profile) (<NL>)*)* <RBRACE>
+}
+
+void firstPhaseItem(RankProfile profile) :
+{
+  String expression;
+  int rerankCount;
+  double dropLimit;
+}
+{
+    ( expression = expression()                        { profile.setFirstPhaseRanking(expression); }
+      | (<KEEPRANKCOUNT> <COLON> rerankCount = integer()) { profile.setKeepRankCount(rerankCount); }
+      | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); }
+    )
+}
+
+/**
+ * Consumes the second-phase block of a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void secondPhase(RankProfile profile) : { }
+{
+    <SECONDPHASE> lbrace() (secondPhaseItem(profile) (<NL>)*)* <RBRACE>
+}
+
+/**
+ * Consumes a statement for a second-phase block.
+ *
+ * @param profile The rank profile to modify.
+ */
+void secondPhaseItem(RankProfile profile) :
+{
+    String expression;
+    int rerankCount;
+}
+{
+    ( expression = expression()                 { profile.setSecondPhaseRanking(expression); }
+      | (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); }
+    )
+}
+
+/**
+ * This rule consumes a summary-features block of a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void summaryFeatures(RankProfile profile) :
+{
+    String features;
+    String inherited = null;
+}
+{
+    ( <SUMMARYFEATURES_SL> { features = token.image.substring(token.image.indexOf(":") + 1).trim(); } |
+      <SUMMARYFEATURES_ML> { features = token.image.substring(token.image.indexOf("{") + 1,
+                                                              token.image.lastIndexOf("}")).trim(); } |
+      <SUMMARYFEATURES_ML_INHERITS> {
+          int inheritsIndex = token.image.indexOf("inherits ");
+          String rest = token.image.substring(inheritsIndex + "inherits ".length());
+          profile.setInheritedSummaryFeatures(rest.substring(0, rest.indexOf(" ")).trim());
+          features = token.image.substring(token.image.indexOf("{") + 1, token.image.lastIndexOf("}")).trim();
+      }
+    )
+    {
+        profile.addSummaryFeatures(getFeatureList(features));
+    }
+}
+
+/**
+ * This rule consumes a match-features block of a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void matchFeatures(RankProfile profile) :
+{
+    String features;
+}
+{
+    ( <MATCHFEATURES_SL> { features = token.image.substring(token.image.indexOf(":") + 1).trim(); } |
+      <MATCHFEATURES_ML> { features = token.image.substring(token.image.indexOf("{") + 1,
+                                                              token.image.lastIndexOf("}")).trim(); } |
+      <MATCHFEATURES_ML_INHERITS> {
+          int inheritsIndex = token.image.indexOf("inherits ");
+          String rest = token.image.substring(inheritsIndex + "inherits ".length());
+          profile.setInheritedMatchFeatures(rest.substring(0, rest.indexOf(" ")).trim());
+          features = token.image.substring(token.image.indexOf("{") + 1, token.image.lastIndexOf("}")).trim();
+      }
+    )
+    {
+        profile.addMatchFeatures(getFeatureList(features));
+    }
+}
+
+/** Consumes a rank-features block of a rank profile */
+void rankFeatures(RankProfile profile) :
+{
+    String features;
+}
+{
+    ( <RANKFEATURES_SL> { features = token.image.substring(token.image.indexOf(":") + 1).trim(); } |
+      <RANKFEATURES_ML> { features = token.image.substring(token.image.indexOf("{") + 1,
+                                                           token.image.lastIndexOf("}")).trim(); } )
+    {
+        profile.addRankFeatures(getFeatureList(features));
+    }
+}
+
+/**
+ * This rule consumes a ignore-default-rank-features statement for a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void ignoreRankFeatures(RankProfile profile) : { }
+{
+    <IGNOREDEFAULTRANKFEATURES> { profile.setIgnoreDefaultRankFeatures(true); }
+}
+
+/**
+ * This rule consumes a num-threads-per-search statement for a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void numThreadsPerSearch(RankProfile profile) :
+{
+    int num;
+}
+{
+    (<NUMTHREADSPERSEARCH> <COLON> num = integer()) { profile.setNumThreadsPerSearch(num); }
+}
+
+/**
+ * This rule consumes a min-hits-per-thread statement for a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void minHitsPerThread(RankProfile profile) :
+{
+    int num;
+}
+{
+    (<MINHITSPERTHREAD> <COLON> num = integer()) { profile.setMinHitsPerThread(num); }
+}
+
+/**
+ * This rule consumes a num-search-partitions statement for a rank profile.
+ *
+ * @param profile the rank profile to modify
+ */
+void numSearchPartitions(RankProfile profile) :
+{
+    int num;
+}
+{
+    (<NUMSEARCHPARTITIONS> <COLON> num = integer()) { profile.setNumSearchPartitions(num); }
+}
+
+/**
+ * This rule consumes a num-threads-per-search statement for a rank profile.
+ *
+ * @param profile the rank profile to modify
+ */
+void termwiseLimit(RankProfile profile) :
+{
+    double num;
+}
+{
+    (<TERMWISELIMIT> <COLON> num = consumeFloat()) { profile.setTermwiseLimit(num); }
+}
+/**
+ * This rule consumes a rank-properties block of a rank profile. There is a little trick within this rule to allow the
+ * final rank property to skip the terminating newline token.
+ *
+ * @param profile the rank profile to modify
+ */
+void rankProperties(RankProfile profile) : { }
+{
+    <RANKPROPERTIES> lbrace() (LOOKAHEAD(rankPropertyItem() <COLON> rankPropertyItem() <NL>)
+                               rankProperty(profile) (<NL>)+)* [rankProperty(profile)] <RBRACE>
+}
+
+/**
+ * This rule consumes a single rank property pair for a rank profile.
+ *
+ * @param profile the rank profile to modify
+ */
+void rankProperty(RankProfile profile) :
+{
+    String key, val;
+}
+{
+    key = rankPropertyItem() <COLON> val = rankPropertyItem()
+    { profile.addRankProperty(key, val); }
+}
+
+
+/**
+ * This rule consumes a single rank property for a rank-properties block.
+ *
+ * @return The token image of the consumed item.
+ */
+String rankPropertyItem() :
+{
+    String image, ret = "";
+}
+{
+    ( ( image = identifierWithDash()              { ret += image; }
+        | image = quotedString()                  { ret += image; }
+        | ( "(" | ")" | <DOT> | <COMMA> )         { ret += token.image; } )+ )
+    { return ret; }
+}
+
+/**
+ * This rule consumes a field-weight statement of a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void fieldWeight(RankProfile profile) :
+{
+    Integer num;
+    String name;
+}
+{
+    <WEIGHT> name = identifier() <COLON> num = integer()
+    { profile.addRankSetting(name, RankProfile.RankSetting.Type.WEIGHT, num); }
+}
+
+/**
+ * This rule consumes a rank-type statement of a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void fieldRankType(RankProfile profile) :
+{
+    String name;
+    String type;
+}
+{
+    <RANKTYPE> name = identifier() <COLON> type = identifier()
+    { profile.addRankSetting(name, RankProfile.RankSetting.Type.RANKTYPE, RankType.fromString(type)); }
+}
+
+/**
+ * This rule consumes a rank filter statement of a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void fieldRankFilter(RankProfile profile) :
+{
+    String name;
+}
+{
+    <RANK> name = identifier() <COLON> <FILTER>
+    { profile.addRankSetting(name, RankProfile.RankSetting.Type.PREFERBITVECTOR, Boolean.TRUE); }
+}
+
+/**
+ * This rule consumes part of a rank-degradation statement of a rank profile.
+ */
+void rankDegradationBinSize() :
+{
+    double freq;
+}
+{
+    <RPBINSIZE> <COLON> freq = consumeFloat()
+    { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'doc-frequency' in 'rank-degradation' is deprecated and has no effect."); }
+}
+
+
+/**
+ * This rule consumes part of a rank-degradation statement of a rank profile.
+ */
+void rankDegradationBinLow() :
+{
+    int n;
+}
+{
+    <RPBINLOW> <COLON> n = integer()
+    { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'min-fullrank-docs' in 'rank-degradation' is deprecated and has no effect."); }
+}
+
+
+/**
+ * This rule consumes part of a rank-degradation statement of a rank profile.
+ */
+void rankDegradationPosbinSize() :
+{
+    double avgOcc;
+}
+{
+    <RPPOSBINSIZE> <COLON> avgOcc = consumeFloat()
+    { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'occurrences-per-doc' in 'rank-degradation' is deprecated and has no effect."); }
+}
+
+
+/**
+ * This rule consumes part of a rank-degradation statement of a rank profile.
+ */
+void rankDegradationItem() :
+{
+}
+{
+    ( rankDegradationBinSize()
+      | rankDegradationBinLow()
+      | rankDegradationPosbinSize() )
+}
+
+/**
+ * This rule consumes a rank-degradation statement of a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ */
+void rankDegradation(RankProfile profile) :
+{
+    double freq;
+}
+{
+    ( <RANKDEGRADATIONFREQ> <COLON> freq = consumeFloat()
+    { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'rank-degradation-frequency' in 'rank-profile' is deprecated and has no effect."); }
+      | <RANKDEGRADATION> lbrace() ( rankDegradationItem() (<NL>)*)+ <RBRACE>
+    )
+}
+
+/**
+ * Consumes a set of constants available in ranking expressions in the enclosing profile.
+ */
+void constants(RankProfile profile) :
+{
+    String name;
+}
+{
+    <CONSTANTS> <LBRACE> (<NL>)*
+      ( name = identifier() ( constantValue(profile, name) |
+                              constantTensor(profile, name) ) (<NL>)* )*
+    <RBRACE>
+}
+
+void constantValue(RankProfile profile, String name) :
+{
+    String value;
+}
+{
+    <COLON> value = identifier() { profile.addConstant(name, Value.parse(value)); }
+}
+
+void constantTensor(RankProfile profile, String name) :
+{
+    String tensorString = "";
+    TensorType tensorType = null;
+}
+{
+    <LBRACE> (<NL>)*
+      (( tensorString = tensorValue() |
+         tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.name(), name)) ) (<NL>)* )* <RBRACE>
+    {
+        if (tensorType != null) {
+            profile.addConstantTensor(name, new TensorValue(Tensor.from(tensorType, tensorString)));
+        } else {
+            profile.addConstantTensor(name, new TensorValue(Tensor.from(tensorString)));
+        }
+    }
+}
+
+String constantTensorErrorMessage(String rankProfileName, String constantTensorName) : {}
+{
+    { return "For constant tensor '" + constantTensorName + "' in rank profile '" + rankProfileName + "'"; }
+}
+
+String tensorValue() :
+{
+    String tensor;
+}
+{
+    ( <TENSOR_VALUE_SL> { tensor = token.image.substring(token.image.indexOf(":") + 1); } |
+      <TENSOR_VALUE_ML> { tensor = token.image.substring(token.image.indexOf("{") + 1,
+                                                         token.image.lastIndexOf("}")); } )
+    {
+        return tensor;
+    }
+}
+
+TensorType tensorTypeWithPrefix(String errorMessage) :
+{ TensorType type; }
+{
+    <TYPE> <COLON> type= tensorType(errorMessage)
+    { return type; }
+}
+
+TensorType tensorType(String errorMessage) :
+{
+    String tensorTypeString;
+}
+{
+    ( <TENSOR_TYPE> ) { tensorTypeString = token.image; }
+    {
+        TensorType tensorType;
+        try {
+            tensorType = TensorType.fromSpec(tensorTypeString);
+        } catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException(errorMessage + ": Illegal tensor type spec: " + e.getMessage());
+        }
+        return tensorType;
+    }
+}
+
+void importField(Schema schema) :
+{
+    String fieldRefSpec;
+    String aliasFieldName;
+}
+{
+    <IMPORT> <FIELD> fieldRefSpec = identifier() <AS> aliasFieldName = identifier() lbrace()
+    <RBRACE>
+    {
+        long nDots = Utils.count(fieldRefSpec, '.');
+        if (nDots != 1) {
+            throw new IllegalArgumentException("Illegal field reference spec '" + fieldRefSpec + "': Does not include a single '.'");
+        }
+        int indexOfDot = fieldRefSpec.indexOf('.');
+        String documentReferenceFieldName = fieldRefSpec.substring(0, indexOfDot);
+        String foreignFieldName = fieldRefSpec.substring(indexOfDot + 1);
+        TemporaryImportedFields importedFields = schema.temporaryImportedFields().get();
+        if (importedFields.hasField(aliasFieldName)) {
+            throw new IllegalArgumentException("For schema '" + schema.getName() + "', import field as '" + aliasFieldName + "': Field already imported");
+        }
+        importedFields.add(new TemporaryImportedField(aliasFieldName, documentReferenceFieldName, foreignFieldName));
+    }
+}
+
+
+/**
+ * This rule consumes an expression token and returns its image.
+ *
+ * @return The consumed token image.
+ */
+String expression() :
+{
+    String exp;
+}
+{
+    ( <EXPRESSION_SL> { exp = token.image.substring(token.image.indexOf(":") + 1); } |
+      <EXPRESSION_ML> { exp = token.image.substring(token.image.indexOf("{") + 1,
+                                                    token.image.lastIndexOf("}")); } )
+    { return exp; }
+}
+
+String identifierWithDash() :
+{
+    String identifier;
+}
+{
+    ( identifier = identifier() { return identifier; } )
+    |
+    ( <IDENTIFIER_WITH_DASH> { return token.image; } )
+}
+
+/**
+ * Consumes an identifier. This must be kept in sync with all word tokens that should be parseable as
+ * identifiers.
+ *
+ * @return the identifier string
+ */
+String identifier() : { }
+{
+    ( <ALIAS>
+      | <ALWAYS>
+      | <ANNOTATION>
+      | <ANNOTATIONREFERENCE>
+      | <ARITY>
+      | <ARRAY>
+      | <AS>
+      | <ASCENDING>
+      | <ATTRIBUTE>
+      | <BODY>
+      | <BOLDING>
+      | <BTREE>
+      | <CASED>
+      | <COMPRESSION>
+      | <COMPRESSIONLEVEL>
+      | <COMPRESSIONTHRESHOLD>
+      | <CONTEXT>
+      | <CREATEIFNONEXISTENT>
+      | <DENSEPOSTINGLISTTHRESHOLD>
+      | <DESCENDING>
+      | <DICTIONARY>
+      | <DIRECT>
+      | <DOCUMENT>
+      | <DOCUMENTSUMMARY>
+      | <DOUBLE>
+      | <DYNAMIC>
+      | <ENABLEBITVECTORS>
+      | <ENABLEONLYBITVECTOR>
+      | <EXACT>
+      | <EXACTTERMINATOR>
+      | <FALSE>
+      | <FASTACCESS>
+      | <FASTSEARCH>
+      | <FIELD>
+      | <FIELDS>
+      | <FIELDSET>
+      | <FILE>
+      | <FILTER>
+      | <FIRSTPHASE>
+      | <FULL>
+      | <FUNCTION>
+      | <GRAM>
+      | <HASH>
+      | <HEADER>
+      | <HUGE>
+      | <ID>
+      | <IDENTICAL>
+      | <IDENTIFIER>
+      | <IGNOREDEFAULTRANKFEATURES>
+      | <IMPORT>
+      | <INDEX>
+      | <INDEXING>
+      | <INDEXINGREWRITE>
+      | <INHERITS>
+      | <INTEGER>
+      | <KEEPRANKCOUNT>
+      | <LITERAL>
+      | <LOCALE>
+      | <LONG>
+      | <LOWERBOUND>
+      | <LOWERCASE>
+      | <MACRO>
+      | <MAP>
+      | <MATCH>
+      | <MATCHPHASE>
+      | <MAXFILTERCOVERAGE>
+      | <MAXHITS>
+      | <MTOKEN>
+      | <MUTABLE>
+      | <NEVER>
+      | <NONE>
+      | <NORMAL>
+      | <NORMALIZING>
+      | <OFF>
+      | <ON>
+      | <ONDEMAND>
+      | <ORDER>
+      | <PREFIX>
+      | <PRIMARY>
+      | <PROPERTIES>
+      | <QUATERNARY>
+      | <QUERYCOMMAND>
+      | <RANK>
+      | <MODEL>
+      | <RANKPROFILE>
+      | <RANKPROPERTIES>
+      | <RANKSCOREDROPLIMIT>
+      | <RANKTYPE>
+      | <RAW>
+      | <REFERENCE>
+      | <REMOVEIFZERO>
+      | <RERANKCOUNT>
+      | <SCHEMA>
+      | <SEARCH>
+      | <SECONDARY>
+      | <SECONDPHASE>
+      | <SORTING>
+      | <SOURCE>
+      | <PAGED>
+      | <SSCONTEXTUAL>
+      | <SSOVERRIDE>
+      | <SSTITLE>
+      | <SSURL>
+      | <STATIC>
+      | <STEMMING>
+      | <STRENGTH>
+      | <STRING>
+      | <STRUCT>
+      | <SUBSTRING>
+      | <SUFFIX>
+      | <SUMMARY>
+      | <SUMMARYTO>
+      | <SYMMETRIC>
+      | <TERTIARY>
+      | <TEXT>
+      | <TO>
+      | <TRUE>
+      | <TYPE>
+      | <UCA>
+      | <UNCASED>
+      | <URI>
+      | <UPPERBOUND>
+      | <USEDOCUMENT>
+      | <VARIABLE>
+      | <WEIGHT>
+      | <WEIGHTEDSET>
+      | <WORD>
+      | <INLINE>
+      | <CONSTANTS>
+    )
+    { return token.image; }
+}
+
+/**
+ * Consumes a string token and returns the token image.
+ *
+ * @return The consumed token image.
+ */
+String string() : { }
+{
+    <STRING> { return token.image; }
+}
+
+/**
+ * Consumes a quoted string token and returns the token image minus the quotes. This does not perform
+ * unescaping of the content, it simply removes the first and last character of the image. However, the token itself can
+ * contain anything but a double quote.
+ *
+ * @return The unquoted token image.
+ */
+String quotedString() : { }
+{
+    <QUOTEDSTRING> { return token.image.substring(1, token.image.length() - 1); }
+}
+
+/**
+ * This rule consumes a boolean value.
+ *
+ * @return The consumed boolean value.
+ */
+Boolean bool() : { }
+{
+    ( ( <ON>  | <TRUE> )  { return true; } |
+      ( <OFF> | <FALSE> ) { return false; } )
+}
+
+/**
+ * This rule consumes an integer token and returns its numeric value.
+ *
+ * @return The consumed integer value.
+ */
+int integer() : { }
+{
+    <INTEGER> { return Integer.parseInt(token.image); }
+}
+
+/**
+ * This rule consumes a long or integer token and returns its numeric value.
+ *
+ * @return The consumed long value.
+ */
+long consumeLong() : { }
+{
+    ( <INTEGER> { return Long.parseLong(token.image); } |
+      <LONG>    { return Long.parseLong(token.image.substring(0, token.image.length()-1)); }
+    )
+}
+
+/**
+ * This rule consumes a floating-point token and returns its numeric value.
+ *
+ * @return The consumed value.
+ */
+double consumeFloat() : { }
+{
+    <DOUBLE> { return Double.valueOf(token.image); }
+}
+
+Number consumeNumber() :
+{
+    Number num;
+}
+{
+    (num = consumeFloat() | num = consumeLong()) { return num; }
+}
+
+/**
+ * This rule consumes an opening brace with leading and trailing newline tokens.
+ */
+void lbrace() : { }
+{
+    (<NL>)* <LBRACE> (<NL>)*
+}