path: root/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf

// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
/**
 * Vespa schema file grammar.
 * NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats it like zero-to-many)
 *
 * @author Shahar Ariel
 * @author bratseth
 */

{
    parserClass="ai.vespa.intellij.schema.parser.SdParser" // Name and the location of the parser which will be generated.

    extends="com.intellij.extapi.psi.ASTWrapperPsiElement" // All nodes will extend this class. Wraps AST node to a PSI node.
    
    // Prefix and suffix for all generated classes
    psiClassPrefix="Sd"
    psiImplClassSuffix="Impl"
    
    psiPackage="ai.vespa.intellij.schema.psi" // Location to be used when generating PSI classes.
    psiImplPackage="ai.vespa.intellij.schema.psi.impl" // Location to be used when generating PSI implementation classes.
       
    elementTypeHolderClass="ai.vespa.intellij.schema.psi.SdTypes" // Element type holder class name.
    
    elementTypeClass="ai.vespa.intellij.schema.psi.SdElementType" // Class which will be used to create internal nodes.
    tokenTypeClass="ai.vespa.intellij.schema.psi.SdTokenType" // Class which will be used to create leaf nodes.
      
    extends(".*Expr")=RankingExpression  // Here to deal with left-recursion that happens in expressions

    // NOTE: As far as I (Jon) can tell these are not used. Edit the ones in sd.flex instead.
    tokens = [
        ID_REG = 'regexp:[a-zA-Z_][a-zA-Z0-9_]*'
        WHITE_SPACE = 'regexp:\s+'
        COMMENT = 'regexp:#.*'
        SYMBOL = 'regexp:[!$|:{}().\[\]]'
        COMMA = 'regexp:[,]'
        INTEGER_REG = 'regexp:[0-9]+'
        FLOAT_REG = 'regexp:[0-9]+[.][0-9]+([eE][-]?[0-9]+)?'
        STRING_REG = 'regexp:\"([^\"\\]*(\\.[^\"\\]*)*)\"'
        STRING_REG_SINGLE_QUOTE = "regexp:'([^'\\]*(\\.[^'\\]*)*)'"
        WORD_REG = 'regexp:\w+'
    ]
}

SdFile ::= NL* ( SchemaDefinition | DocumentDefinition | RankProfileDefinition )
SchemaDefinition ::= (search | schema) IdentifierVal? NL* (inherits IdentifierVal)? BlockStart SchemaBody BlockEnd
SchemaBody ::= ( SchemaBodyOptions | NL )* DocumentDefinition ( SchemaBodyOptions | NL )* // Does not support zero-or-one occurrences
private SchemaBodyOptions ::= SchemaFieldDefinition | ImportFieldDefinition | DocumentSummaryDefinition | 
                              RankProfileDefinition | IndexDefinition | DocumentStructDefinition |
                              FieldSetDefinition | ConstantDefinition | OnnxModelDefinition | StemmingDefinition |
                              ( raw-as-base64-in-summary ':' (true | false) ) | SchemaAnnotationDefinition

         
SchemaFieldDefinition ::= field IdentifierVal type FieldTypeName BlockStart SchemaFieldBody BlockEnd
                          { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                            implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                           }
                            
FieldTypeName ::= ("array" '<' (FieldTypeName | IdentifierVal) '>') | ("weightedset" '<' SingleValueFieldTypeName '>') | 
                  ("map" '<' (FieldTypeName | IdentifierVal) COMMA (FieldTypeName | IdentifierVal) '>') | TensorType |
                  (SingleValueFieldTypeName '[' ']') | SingleValueFieldTypeName
private SingleValueFieldTypeName ::= "string" | "int" | "long" | "bool" | "byte" | "float" | "double" | "position" | "predicate" | "raw" | "uri" | 
                             "reference" '<' IdentifierVal '>' | "annotationreference" '<' IdentifierVal '>' | IdentifierVal
private TensorType ::= "tensor" ('<' ("float" | "double" | "int8" | "bfloat16") '>')? '(' TensorDimension (COMMA TensorDimension)* ')'
private TensorDimension ::= WordWrapper (('{' '}') | ('[' INTEGER_REG ']'))

SchemaFieldBody ::= ( DocumentFieldBodyOptions | NL )* // Fields of schemas and documents defined the same way here

DocumentSummaryDefinition ::= document-summary IdentifierWithDashVal (inherits IdentifierWithDashVal (COMMA IdentifierWithDashVal)*)? BlockStart DocumentSummaryBody BlockEnd
                              { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                                implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                               }
DocumentSummaryBody ::= ( DocumentSummaryBodyOptions NL* )* // Does not support zero-or-one occurrences
private DocumentSummaryBodyOptions ::= SummaryDefinition | omit-summary-features | from-disk

ImportFieldDefinition ::= import field IdentifierVal '.' IdentifierVal as IdentifierVal BlockStart BlockEnd
                          { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                            implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                           }

FieldSetDefinition ::= fieldset IdentifierVal BlockStart FieldSetBody BlockEnd
FieldSetBody ::= ( FieldSetBodyOptions | NL )*
private FieldSetBodyOptions ::= (fields ':' DottedIdentifier (COMMA DottedIdentifier)*) | QueryCommandDefinition | MatchDefinition

ConstantDefinition ::= constant IdentifierVal BlockStart ConstantBody BlockEnd
ConstantBody ::= ( ConstantBodyOptions | NL )*
private ConstantBodyOptions ::= (file ':' FilePath) | (uri ':' UriPath) | (type ':' TensorType)
private FilePath ::= ( WordWrapper | INTEGER_REG) (('.' | '/' | '-') ( WordWrapper | INTEGER_REG) )+
private UriPath ::= ('H'|'h') ('T'|'t') ('T'|'t') ('P'|'p') ('S'|'s')? ':' ('//')? (IdentifierWithDashVal | '.' | '/' | ':')+


OnnxModelDefinition ::= onnx-model IdentifierVal BlockStart OnnxModelBody BlockEnd
OnnxModelBody ::= ( OnnxModelBodyOptions | NL )*
private OnnxModelBodyOptions ::= (file ':' FilePath) | (uri ':' UriPath) | 
                                 ((input | output) (RankFeature | IdentifierVal | STRING_REG) ':' ('.' | '/' | '(' | ')' | IdentifierWithDashVal | WORD_REG))

SchemaAnnotationDefinition ::= AnnotationDefinition
                        { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                          implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                         }
                          
private AnnotationDefinition ::= annotation IdentifierVal (inherits IdentifierVal)? BlockStart AnnotationFieldDefinition* BlockEnd
AnnotationFieldDefinition ::= field IdentifierVal type FieldTypeName BlockStart BlockEnd
                          { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                            implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                           }

//-------------------------
//    Expressions rules
//
//    NOTE: These must end by "Expr" - see this line above: extends(".*Expr")=RankingExpression
//          The *Expr alternatives are consumed greedily so order matters.
//-------------------------
RankingExpression ::= RenameExpr | LiteralOrGenerateTensorExpr | FilePathExpr | ParenthesisedExpr | BinaryExpr | UnaryExpr |
                      IfFunctionExpr | FunctionCallOrLambdaExpr | RankFeatureExpr | InListRankingExpr | PrimitiveExpr | SliceExpr

FilePathExpr ::= file ':' (FilePath | WordWrapper)

IfFunctionExpr ::= "if" ClauseStart RankingExpression CommaSeparator RankingExpression CommaSeparator RankingExpression ClauseEnd

InListRankingExpr ::= RankingExpression "in" '[' RankingExpression (CommaSeparator RankingExpression)* ']'

BinaryExpr ::= RankingExpression NL* ( ArithmeticOperator | ComparisonOperator) NL* RankingExpression
ArithmeticOperator ::= '+' | '-' | '*' | '/' | '%' | '^' | "||" | "&&"
ComparisonOperator ::= '<' | '>' | "==" | "<=" | ">=" | "~=" | "!="

UnaryExpr ::= UnaryOperator NL* RankingExpression
UnaryOperator ::= '!' | '-'

RankFeatureExpr ::= RankFeature

// The rename function allows lists of identifiers as arguments
RenameExpr ::= "rename" ClauseStart RankingExpression CommaSeparator
                                    ( ArgumentDeclarations | ArgumentDeclaration ) CommaSeparator
                                    ( ArgumentDeclarations | ArgumentDeclaration )
                        ClauseEnd

// Rough parsing but hard to do better due to greediness: If this is a lambda arg expressions must be identifiers
FunctionCallOrLambdaExpr ::= IdentifierVal ClauseStart RankingExpression ( CommaSeparator RankingExpression )* ClauseEnd ( '.' IdentifierVal )?
                             (ParenthesisedExpr)? // This turns the function call into a lambda
 
ParenthesisedExpr ::= ClauseStart RankingExpression ClauseEnd

PrimitiveExpr ::= ( AnyNumber | ('$')?IdentifierVal | RankFeature | STRING_REG | STRING_REG_SINGLE_QUOTE )

SliceExpr ::= RankingExpression ( SliceKey | SliceIndex | FullTensorAddress )

SliceKey ::= BlockStart Label BlockEnd
SliceIndex ::= '[' RankingExpression ']'

FullTensorAddress ::= BlockStart KeyValue ( CommaSeparator KeyValue )* BlockEnd

KeyValue ::= IdentifierVal ':' ( RankingExpression | Label )

Label ::= IdentifierVal | STRING_REG | STRING_REG_SINGLE_QUOTE

LiteralOrGenerateTensorExpr ::= TensorType (
                                  ( NL* ':' NL* TensorValue ) | // literal verbose form tensor
                                  ParenthesisedExpr // generate tensor
                                )

TensorValue ::= MappedTensorValue | ArrayTensorValues
MappedTensorValue ::= BlockStart MappedTensorBlock ( COMMA MappedTensorBlock )* BlockEnd
MappedTensorBlock ::= TensorAddress ':' ( TensorCellValue | ArrayTensorValues )
ArrayTensorValues ::= NL* '[' NL* ( TensorCellValue | ArrayTensorValues ) ( CommaSeparator ( TensorCellValue | ArrayTensorValues ) )* NL* ']' NL*
TensorAddress ::= Label | FullTensorAddress
TensorCellValue ::= RankingExpression

//-------------------------
//-- Rank Profile rules ---
//-------------------------
RankProfileDefinition ::= (rank-profile | model) IdentifierWithDashVal (inherits IdentifierWithDashVal (COMMA IdentifierWithDashVal)*)? BlockStart RankProfileBody BlockEnd
                          { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                            implements=["ai.vespa.intellij.schema.psi.SdDeclaration"]
                           }
private RankProfileBody ::= ( RankProfileBodyOptions | NL )*
private RankProfileBodyOptions ::= MatchPhaseDefinition | NumThreadsDefinition | FunctionDefinition | TermwiseLimitDefinition | 
                           ignore-default-rank-features | RankPropertiesDefinition | FirstPhaseDefinition | 
                           SummaryFeaturesDefinition | MatchFeaturesDefinition | RankFeaturesDefinition | 
                           SecondPhaseDefinition | ConstantsDefinition | RankDefinition | RankTypeDefinition | 
                           MinHitsDefinition | NumSearchPartitionsDefinition | FieldWeightDefinition | StrictDefinition |
                           InputsDefinition

MatchPhaseDefinition ::= match-phase BlockStart MatchPhaseBody BlockEnd
MatchPhaseBody ::= ( MatchPhaseBodyOptions | NL )*
MatchPhaseBodyOptions ::= (attribute ':' DottedIdentifier) | (order ':' (ascending | descending)) | (max-hits ':' INTEGER_REG)
                          | DiversityDefinition | (evaluation-point ':' FLOAT_REG) |
                          (pre-post-filter-tipping-point ':' FLOAT_REG) // Does not support zero-or-one occurrences
DiversityDefinition ::= diversity BlockStart DiversityBody BlockEnd
DiversityBody ::= ( DiversityBodyOptions | NL )*
private DiversityBodyOptions ::= (attribute ':' DottedIdentifier) | (min-groups ':' INTEGER_REG) | (cutoff-factor ':' FLOAT_REG) |
                                 (cutoff-strategy ':' (strict | loose))

private NumThreadsDefinition ::= num-threads-per-search ':' INTEGER_REG
private TermwiseLimitDefinition ::= termwise-limit ':' (FLOAT_REG | INTEGER_REG)
private MinHitsDefinition ::= min-hits-per-thread ':' INTEGER_REG
private NumSearchPartitionsDefinition ::= num-search-partitions ':' INTEGER_REG
FieldWeightDefinition ::= weight DottedIdentifier ':' INTEGER_REG
StrictDefinition ::= strict ':' (true | false)
InputsDefinition ::= inputs BlockStart ( InputDefinition | NL )* BlockEnd
InputDefinition ::= ( QueryFeature | IdentifierVal)
                    (':')?
                    ( TensorType | "double" )? (':'? ( AnyNumber | TensorValue ) )?

FirstPhaseDefinition ::= first-phase BlockStart FirstPhaseBody BlockEnd
                         { mixin="ai.vespa.intellij.schema.psi.impl.SdFirstPhaseDefinitionMixin" }
FirstPhaseBody ::= ( FirstPhaseBodyOptions | NL )* // Does not support zero-or-one occurrences
private FirstPhaseBodyOptions ::= (keep-rank-count ':' INTEGER_REG) | (rank-score-drop-limit ':' AnyNumber) | ExpressionDefinition

ExpressionDefinition ::= expression ((':' RankingExpression) | (BlockStart RankingExpression* BlockEnd))

SecondPhaseDefinition ::= second-phase BlockStart SecondPhaseBody BlockEnd
SecondPhaseBody ::= ( SecondPhaseBodyOptions | NL )*
private SecondPhaseBodyOptions ::= (rerank-count ':' INTEGER_REG) | ExpressionDefinition

RankPropertiesDefinition ::= rank-properties BlockStart RankPropertiesBody BlockEnd
RankPropertiesBody ::= (RankPropertiesKey ':' RankPropertiesValue NL+ )+
RankPropertiesKey ::= (IdentifierWithDashVal | STRING_REG | '(' | ')' | '.' | COMMA | '$' | INTEGER_REG)+
RankPropertiesValue ::= AnyNumber | WORD_REG | DottedIdentifier | STRING_REG

FunctionDefinition ::= (function | macro) inline? IdentifierVal ArgumentDeclarations
                       BlockStart ExpressionDefinition BlockEnd
                       { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                         implements=["ai.vespa.intellij.schema.psi.SdFunctionDefinitionInterface" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                       }

ArgumentDeclarations ::= '()' | ( ClauseStart ArgumentDeclaration (CommaSeparator ArgumentDeclaration)* NL* ClauseEnd )
ArgumentDeclaration ::= IdentifierVal
                       { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                         implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                       }

SummaryFeaturesDefinition ::= summary-features ((':' RankFeature+ NL+) | ((inherits IdentifierWithDashVal)? BlockStart ( RankFeature | NL )* BlockEnd))

MatchFeaturesDefinition ::= match-features ((':' RankFeature+ NL+) | ((inherits IdentifierWithDashVal)? BlockStart ( RankFeature | NL )* BlockEnd))

RankFeaturesDefinition ::= rank-features ((':' RankFeature+ NL+) | (BlockStart ( RankFeature | NL )* BlockEnd))

ConstantsDefinition ::= constants BlockStart ( InnerConstantDefinition | NL )* BlockEnd

InnerConstantDefinition ::= ( ("constant" ClauseStart IdentifierVal ClauseEnd ) | IdentifierVal )
                            (':')? ( TensorType | "double" )?
                            (':')? ( TensorValue | AnyNumber | (file ':' FilePath) | (uri ':' UriPath))

RankFeature ::= ForEachFeature  | ( IdentifierVal (FeatureArguments)? ( '.' IdentifierVal )* )
                    { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                      implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                    }
QueryFeature ::= "query" '(' IdentifierWithDashVal ')'
                    { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                      implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                    }

// Needs special handling due to the nested rank feature, and the condition
ForEachFeature ::= "foreach" ClauseStart
                     IdentifierVal CommaSeparator                                                 // dimension
                     IdentifierVal CommaSeparator                                                 // variable
                     RankFeature CommaSeparator                                                   // feature
                     ( ('>' AnyNumber) | ('<'  AnyNumber) | "true" | STRING_REG )  CommaSeparator // condition
                     IdentifierVal                                                                // operation
                   ClauseEnd

// Not using ClauseEnd here to avoid consuming trailing NL's, which must be left to terminate single-line
// rank feature lists in summary/match/rank-features
FeatureArguments ::= '()' | ( ClauseStart FeatureArgument (CommaSeparator FeatureArgument)* NL* ')' )
FeatureArgument ::= DottedIdentifier | INTEGER_REG | STRING_REG | STRING_REG_SINGLE_QUOTE
                       { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                         implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                       }

//-------------------------
//---- Document rules -----
//-------------------------
DocumentDefinition ::= document (IdentifierVal (inherits IdentifierVal (COMMA IdentifierVal)*)?)? BlockStart DocumentBody BlockEnd
                      { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                        implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                      }

DocumentBody ::= ( DocumentBodyOptions | NL )*
DocumentBodyOptions ::= DocumentStructDefinition | DocumentFieldDefinition | DocumentAnnotationDefinition

DocumentAnnotationDefinition ::= AnnotationDefinition
                        { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                          implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                        }

DocumentStructDefinition ::= struct IdentifierVal (inherits IdentifierVal)? BlockStart DocumentStructBody BlockEnd
                             { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                               implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                             }

DocumentStructBody ::= ( DocumentStructFieldDefinition | NL )*
DocumentStructFieldDefinition ::= field IdentifierVal type FieldTypeName BlockStart DocumentStructFieldBody BlockEnd
                                  { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                                    implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                                  }
DocumentStructFieldBody ::= ( MatchDefinition NL* )?

DocumentFieldDefinition ::= field IdentifierVal type FieldTypeName BlockStart DocumentFieldBody BlockEnd
                           { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                             implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                           }

DocumentFieldBody ::= ( DocumentFieldBodyOptions | NL )* // Does not support zero-or-one occurrences
private DocumentFieldBodyOptions ::= StructFieldDefinition | MatchDefinition | IndexingDefinition | AttributeDefinition | 
                             AliasDefinition | RankDefinition | IndexingRewriteState | QueryCommandDefinition | SummaryDefinition |
                             BoldingDefinition | (id ':' INTEGER_REG) | IndexDefinition | (normalizing ':' IdentifierWithDashVal) | 
                             SortingDefinition | StemmingDefinition | (weight ':' INTEGER_REG) | WeightedSetDefinition |
                             RankTypeDefinition | DictionaryDefinition | SummaryToDefinition | header | body

// Struct
StructFieldDefinition ::= struct-field DottedIdentifier BlockStart StructFieldBody BlockEnd
                          { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
                            implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
                          }

StructFieldBody ::= ( StructFieldBodyOptions | NL )* // Does not support zero-or-one occurrences
StructFieldBodyOptions ::= SummaryDefinition | IndexingDefinition | AttributeDefinition | MatchDefinition |
                           QueryCommandDefinition | StructFieldDefinition | RankDefinition

// Match
MatchDefinition ::= match ((':' MatchProperty NL*) | ( BlockStart ( MatchProperty NL* )* BlockEnd))
MatchProperty ::= text | token | exact | (exact-terminator ':' STRING_REG) | word | prefix | cased | uncased | substring |
                  suffix | (max-length ':' INTEGER_REG) | gram | (gram-size ':' INTEGER_REG) | WordWrapper
// Indexing
IndexingDefinition ::= indexing ((':' IndexingStatement NL+) | ( BlockStart ( IndexingStatement ';' NL* )* BlockEnd))
IndexingStatement ::= IndexingExpression (('|' IndexingExpression)*)
IndexingExpression ::= (input ( DottedIdentifier | IndexingStuff )+ )+ |
                       summary DottedIdentifier? | attribute DottedIdentifier? | index DottedIdentifier? |
                       set_language | lowercase |
                       IndexingStuff+ |
                       ( WordWrapper? BlockStart (NL | IndexingStatement)* BlockEnd) |
// TODO: Parse advanced indexing statements properly
private IndexingStuff ::= WordWrapper | AnyNumber | STRING_REG | BlockStart | BlockEnd |
                          ':' | '.' | '(' | ')' | ArithmeticOperator | ComparisonOperator

// Attribute
AttributeDefinition ::= attribute ((':' SimpleAttributeProperty NL+ ) |
                                   BlockStart (ComplexAttributeProperty | SimpleAttributeProperty | NL)+ BlockEnd )
SimpleAttributeProperty ::= fast-search | fast-rank | fast-access | paged | mutable | enable-bit-vectors | enable-only-bit-vector | WordWrapper
ComplexAttributeProperty ::= AliasDefinition | SortingDefinition | DistanceMetricDef
DistanceMetricDef ::= distance-metric ':' IdentifierWithDashVal NL+
AliasDefinition ::= alias DottedIdentifier? ':' DottedIdentifierWithDash NL+
StemmingDefinition ::= stemming ':' IdentifierWithDashVal NL+
RankDefinition ::= rank ((DottedIdentifier? ':' RankingSetting NL+) | (BlockStart ( RankingSetting | NL )+ BlockEnd))
RankingSetting ::= filter | normal | literal | WordWrapper
IndexingRewriteState ::= indexing-rewrite ':' none NL+
QueryCommandDefinition ::= query-command ':' (IdentifierWithDashVal | STRING_REG | WordWrapper) NL+
// Summary
SummaryDefinition ::= summary IdentifierWithDashVal? (type FieldTypeName)? ((':' SummaryBodyOptions NL+ ) | ( BlockStart SummaryBody BlockEnd))
                      { mixin="ai.vespa.intellij.schema.psi.impl.SdSummaryDefinitionMixin" }
SummaryBody ::= ( SummaryBodyOptions | NL )* // Does not support zero-or-one occurrences
SummaryBodyOptions ::= full | static | dynamic | ((source ':' DottedIdentifier) (COMMA DottedIdentifier)*) |
                       (to ':' IdentifierVal (COMMA IdentifierVal)*) | matched-elements-only | BoldingDefinition
SummaryToDefinition ::= summary-to ':' WordWrapper (COMMA WordWrapper)* NL+
BoldingDefinition ::= bolding ':' (on | off | true | false) NL+

// Index
IndexDefinition ::= index IdentifierVal? ((':' IndexProperty NL+ ) | (BlockStart ( IndexProperty | NL)* BlockEnd))
private IndexProperty ::= (alias ':' DottedIdentifierWithDash) | StemmingDefinition | (arity ':' INTEGER_REG) |
                          (lower-bound ':' INTEGER_REG ('L')?) | (upper-bound ':' INTEGER_REG ('L')?) |
                          (dense-posting-list-threshold ':' FLOAT_REG) | enable-bm25 | prefix | HnswDefinition
HnswDefinition ::= hnsw BlockStart HnswBody BlockEnd
HnswBody ::= ( HnswBodyOptions | NL )*
private HnswBodyOptions ::= (max-links-per-node ':' INTEGER_REG) | (neighbors-to-explore-at-insert ':' INTEGER_REG) |
                            (multi-threaded-indexing ':' (on | off | true | false))
// Sorting
SortingDefinition ::= sorting ((':' SortingProperty (NL)+) | (BlockStart ( SortingProperty | NL)* BlockEnd))
SortingProperty ::= ascending | descending | (function ':' SortingFunction) | (strength ':' SortingStrength) | 
                    (locale ':' IdentifierWithDashVal)
SortingFunction ::= uca | raw | lowercase
SortingStrength ::= primary | secondary | tertiary | quaternary | identical
// Rank Type
RankTypeDefinition ::= rank-type IdentifierVal? ':' IdentifierVal NL+
// Weighted Set
WeightedSetDefinition ::= weightedset ((':' WeightedSetProperty NL+ ) | (BlockStart ( WeightedSetProperty | NL )* BlockEnd))
WeightedSetProperty ::=  create-if-nonexistent | remove-if-zero
// Dictionary
DictionaryDefinition ::= dictionary ((':' DictionarySetting NL+) | (BlockStart ( DictionarySetting | NL )* BlockEnd))
DictionarySetting ::= hash | btree | cased | uncased

private WordWrapper ::= KeywordOrIdentifier | KeywordNotIdentifier | ID_REG | IdentifierWithDashVal | WORD_REG

IdentifierVal ::= KeywordOrIdentifier | ID_REG { mixin="ai.vespa.intellij.schema.psi.impl.SdIdentifierMixin"
                                                 implements=["ai.vespa.intellij.schema.psi.SdIdentifier"]
                                               }
DottedIdentifier ::= IdentifierVal ('.' IdentifierVal)*

IdentifierWithDashVal ::= IdentifierVal ('-' ( IdentifierVal | INTEGER_REG ) )*
                                               { mixin="ai.vespa.intellij.schema.psi.impl.SdIdentifierMixin"
                                                 implements=["ai.vespa.intellij.schema.psi.SdIdentifier"]
                                               }
DottedIdentifierWithDash ::= IdentifierWithDashVal ('.' IdentifierWithDashVal )*

BlockStart ::= NL* '{' NL*
BlockEnd ::=   NL* '}' NL*
ClauseStart ::= NL* '(' NL*
ClauseEnd ::=   NL* ')' NL*
CommaSeparator ::= NL* COMMA NL*
AnyNumber ::= ('-')? ( FLOAT_REG | INTEGER_REG)

// Those lists of keywords (KeywordOrIdentifier and KeywordNotIdentifier) have to be synchronized with sd.flex file.
// If you add a keyword here, you should add it to the sd.flex file as well.
KeywordOrIdentifier ::= schema | search | document | struct | field | type | indexing | input | output | inherits | 
                        import | as | raw | uri | file | annotationreference | array | weightedset | map |
                        order | ascending | descending | diversity | constants | expression | weight | match |
                        function | macro | inline | text | exact | word | prefix | cased | uncased | substring | suffix | 
                        gram | paged | mutable | alias | sorting | strength | locale | uca | lowercase |
                        primary | secondary | tertiary | quaternary | identical | rank | filter | normal | literal | 
                        none | full | dynamic | source | to | strict | loose |
                        bolding | on | off | true | false | id | normalizing | stemming | arity | hnsw | dictionary | hash | btree |
                        fieldset | fields | constant | annotation
                        | attribute | body | header | index | static |
                        reference | summary | set_language | model

// Note- in this form, those keywords can't be use as identifier-with-dash!
KeywordNotIdentifier ::= struct-field | document-summary | omit-summary-features | from-disk | rank-profile | rank-type |
                         num-threads-per-search | termwise-limit | ignore-default-rank-features | min-hits-per-thread | 
                         num-search-partitions | match-phase | max-hits | second-phase | rerank-count | min-groups |
                         first-phase | keep-rank-count | rank-score-drop-limit | rank-properties | summary-features | 
                         match-features | rank-features |
                         exact-terminator | max-length | gram-size | fast-search | fast-rank | fast-access | distance-metric |
                         indexing-rewrite | query-command | matched-elements-only | lower-bound | upper-bound | 
                         dense-posting-list-threshold | enable-bm25 | max-links-per-node | neighbors-to-explore-at-insert | 
                         multi-threaded-indexing | create-if-nonexistent | remove-if-zero | raw-as-base64-in-summary |
                         onnx-model | cutoff-factor | cutoff-strategy | on-match | on-rank | on-summary | enable-bit-vectors |
                         enable-only-bit-vector | summary-to | evaluation-point | pre-post-filter-tipping-point