Grammar fixes- all .sd test files until function_arguments_with_expressions/test.sd

3 files changed, 47 insertions, 36 deletions

diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java
index 7ff108fc8f6..851c660d9f5 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java
+++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java
@@ -172,6 +172,7 @@ public class SdSyntaxHighlighter extends SyntaxHighlighterBase {
         constants.add(SdTypes.SUMMARY);
         constants.add(SdTypes.INDEX);
         constants.add(SdTypes.SET_LANGUAGE);
+        constants.add(SdTypes.LOWERCASE);
         constants.add(SdTypes.FAST_SEARCH);
         constants.add(SdTypes.FAST_ACCESS);
         constants.add(SdTypes.PAGED);
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex b/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex
index 235b494cb0e..0d5c62fce73 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex
+++ b/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex
@@ -32,7 +32,7 @@ ID_WITH_DASH = [a-zA-Z_][a-zA-Z0-9_-]*
 WHITE_SPACE=[ \t\n\x0B\f\r]+
 
 COMMENT=#.*
-SYMBOL= [|:{}(),.\[\]]
+SYMBOL= [!|:{}(),.\[\]]
 INTEGER = [0-9]+
 FLOAT = {INTEGER}[.][0-9]+[e]?
 COMPARISON_OPERATOR = [<>]|(==)|(<=)|(>=)|(\~=)
@@ -109,17 +109,20 @@ WORD = \w+
   "rank"                     { return RANK; }
   "filter"                   { return FILTER; }
   "normal"                   { return NORMAL; }
+  "literal"                  { return LITERAL; }
   "indexing-rewrite"         { return INDEXING_REWRITE; }
   "none"                     { return NONE; }
   "query-command"            { return QUERY_COMMAND; }
   "full"                     { return FULL; }
-  "dinamic"                  { return DYNAMIC; }
+  "dynamic"                  { return DYNAMIC; }
   "source"                   { return SOURCE; }
   "to"                       { return TO; }
   "matched-elements-only"    { return MATCHED_ELEMENTS_ONLY; }
 
   "input"                    { return INPUT; }
   "mutable"                  { return MUTABLE; }
+  "enable-bit-vectors"       { return ENABLE_BIT_VECTORS; }
+  "enable-only-bit-vector"   { return ENABLE_ONLY_BIT_VECTOR; }
   "document-summary"         { return DOCUMENT_SUMMARY; }
   "from-disk"                { return FROM_DISK; }
   "omit-summary-features"    { return OMIT_SUMMARY_FEATURES; }
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf b/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf
index c8c7af8ade8..1d71597729b 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf
+++ b/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf
@@ -28,7 +28,7 @@ NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats
         ID_WITH_DASH_REG = 'regexp:[a-zA-Z_][a-zA-Z0-9_-]*'
         WHITE_SPACE = 'regexp:\s+'
         COMMENT = 'regexp:#.*'
-        SYMBOL = 'regexp:[|:{}(),.\[\]]'
+        SYMBOL = 'regexp:[!|:{}(),.\[\]]'
         COMPARISON_OPERATOR = 'regexp:[<>]|(==)|(<=)|(>=)|(~=)'
         ARITHMETIC_OPERATOR = 'regexp:[\-+*/]'
         INTEGER_REG = 'regexp:[0-9]+'
@@ -54,18 +54,19 @@ SchemaFieldDefinition ::= field IdentifierVal type FieldTypeName '{' SchemaField
                             
 FieldTypeName ::= ("array" '<' (FieldTypeName | IdentifierVal) '>') | ("weightedset" '<' SingleValueFieldTypeName '>') | 
                   ("map" '<' (FieldTypeName | IdentifierVal) ',' (FieldTypeName | IdentifierVal) '>') | TensorType |
-                  SingleValueFieldTypeName
+                  (SingleValueFieldTypeName '[' ']') | SingleValueFieldTypeName
 private SingleValueFieldTypeName ::= "string" | "int" | "long" | "bool" | "byte" | "float" | "double" | "position" | "predicate" | "raw" | "uri" | 
                              "reference" '<' IdentifierVal '>' | "annotationreference" '<' IdentifierVal '>' | IdentifierVal
 private TensorType ::= "tensor" '<' ("float" | "double" | "int8" | "bfloat16") '>' '(' TensorDimension (',' TensorDimension)* ')'
-private TensorDimension ::= WORD_REG ('{' '}') | ('[' INTEGER_REG ']')
+private TensorDimension ::= WordWrapper ('{' '}') | ('[' INTEGER_REG ']')
 
-SchemaFieldBody ::= SchemaFieldBodyOptions* // Does not support zero-or-one occurrences
-SchemaFieldBodyOptions ::= SchemaFieldIndexingDefinition | AttributeDefinition | RankDefinition | IndexingRewriteState |
-                           MatchDefinition | StructFieldDefinition | QueryCommandDefinition               
-                    
-SchemaFieldIndexingDefinition ::= indexing (':' SchemaFieldIndexingStatement) | ('{' SchemaFieldIndexingStatement+ '}')
-SchemaFieldIndexingStatement ::= (input IdentifierVal) | IndexingStatement 
+SchemaFieldBody ::= DocumentFieldBodyOptions* // SchemaFieldBodyOptions* // Does not support zero-or-one occurrences
+//SchemaFieldBodyOptions ::= SchemaFieldIndexingDefinition | AttributeDefinition | RankDefinition | IndexingRewriteState |
+//                           MatchDefinition | StructFieldDefinition | QueryCommandDefinition | StemmingDefinition | RankTypeDefinition |
+//                           AliasDef 
+//                    
+//SchemaFieldIndexingDefinition ::= indexing ((':' SchemaFieldIndexingStatement) | ('{' SchemaFieldIndexingStatement+ '}'))
+//SchemaFieldIndexingStatement ::= IndexingStatement //|  | IndexingStatement // ('|' IndexingStatement)?)
 
 DocumentSummaryDefinition ::= document-summary IdentifierWithDashVal (inherits IdentifierWithDashVal)? '{' DocumentSummaryBody '}' 
                               { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
@@ -100,7 +101,7 @@ SchemaAnnotationDefinition ::= AnnotationDefinition
                           implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
                          }
                           
-private AnnotationDefinition ::= annotation IdentifierVal '{' AnnotationFieldDefinition* '}'
+private AnnotationDefinition ::= annotation IdentifierVal (inherits IdentifierVal)? '{' AnnotationFieldDefinition* '}'
 AnnotationFieldDefinition ::= field IdentifierVal type FieldTypeName '{' '}'
                           { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
                             implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
@@ -109,11 +110,11 @@ AnnotationFieldDefinition ::= field IdentifierVal type FieldTypeName '{' '}'
 //-------------------------
 //--- Expressions rules ---
 //-------------------------
-RankingExpression ::= ParenthesisedExpr | BooleanExpr |ArithmeticExpr | IfFunctionExpr | 
-                      QueryDefinitionExpr | FunctionCallExpr | PrimitiveExpr
+RankingExpression ::= ParenthesisedExpr | BooleanExpr | ArithmeticExpr | IfFunctionExpr | 
+                      QueryDefinitionExpr | FunctionCallExpr | InListRankingExpr | PrimitiveExpr
 
-IfFunctionExpr ::= "if" '(' (InListRankingExpression | RankingExpression) ',' RankingExpression ',' RankingExpression ')'
-InListRankingExpression ::= RankingExpression "in" '[' RankingExpression (',' RankingExpression)* ']'
+IfFunctionExpr ::= "if" '(' (InListRankingExpr | RankingExpression) ',' RankingExpression ',' RankingExpression ')'
+InListRankingExpr ::= RankingExpression "in" '[' RankingExpression (',' RankingExpression)* ']'
 
 BooleanExpr ::= RankingExpression COMPARISON_OPERATOR RankingExpression
                 
@@ -125,7 +126,7 @@ FunctionCallExpr ::= IdentifierWithDashVal '(' RankingExpression (',' RankingExp
  
 ParenthesisedExpr ::= '(' RankingExpression ')'
 
-PrimitiveExpr ::= (('-')? INTEGER_REG) | (('-')? FLOAT_REG) | IdentifierVal | RankFeature
+PrimitiveExpr ::= (('-')? INTEGER_REG) | (('-')? FLOAT_REG) | IdentifierVal | RankFeature | STRING_REG
 
 //-------------------------
 //-- Rank Profile rules ---
@@ -238,7 +239,7 @@ private DocumentFieldBodyOptions ::= StructFieldDefinition | MatchDefinition | I
 
 //***** Field's body elements ******//
 // Struct
-StructFieldDefinition ::= struct-field IdentifierVal '{' StructFieldBody '}'
+StructFieldDefinition ::= struct-field IdentifierVal ('.' IdentifierVal)? '{' StructFieldBody '}'
                           { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
                             implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
                           }
@@ -248,29 +249,32 @@ StructFieldBodyOptions ::= IndexingDefinition | AttributeDefinition | MatchDefin
                            StructFieldDefinition | SummaryDefinition            
 // Match
 MatchDefinition ::= match ((':' MatchProperty) | ('{' MatchProperty+ '}'))
-MatchProperty ::= text | exact | exact-terminator | word | prefix | cased | uncased | substring | suffix | max-length | 
-                  gram | gram-size
+MatchProperty ::= text | exact | (exact-terminator ':' STRING_REG) | word | prefix | cased | uncased | substring | 
+                  suffix | (max-length ':' INTEGER_REG) | gram | (gram-size ':' INTEGER_REG) | WordWrapper
 // Indexing
-IndexingDefinition ::= indexing (':' IndexingStatement) | ('{' IndexingStatement+ '}')
-IndexingStatement ::= IndexingStatementOptions (('|' IndexingStatementOptions)*) | ((';' IndexingStatementOptions)*)
+IndexingDefinition ::= indexing ((':' IndexingStatement) | ('{' IndexingStatement+ '}'))
+IndexingStatement ::= IndexingStatementOptions ((('|' | ';') IndexingStatementOptions)*)
                       // Does not support zero-or-one occurrences
-IndexingStatementOptions ::= summary | attribute | index | set_language
+IndexingStatementOptions ::= summary | attribute | index | set_language | lowercase | (input (IdentifierVal | IndexingStuff)+) | 
+                             ('{' IndexingStatementOptions '}') | IndexingStuff+
+private IndexingStuff ::= WordWrapper | INTEGER_REG | FLOAT_REG | STRING_REG | '{' IndexingStatementOptions+ '}' |
+                          ':' | ('|' IndexingStatementOptions) | ';' | '.' | '(' | ')' | ARITHMETIC_OPERATOR | COMPARISON_OPERATOR
 // Attribute
 AttributeDefinition ::= attribute ((':' SimpleAttributeProperty) | ('{' (SimpleAttributeProperty | ComplexAttributeProperty)+ '}'))
-SimpleAttributeProperty ::= fast-search | fast-access | paged | mutable // Does not support zero-or-one occurrences
+SimpleAttributeProperty ::= fast-search | fast-access | paged | mutable | enable-bit-vectors | enable-only-bit-vector | WordWrapper // Does not support zero-or-one occurrences
 ComplexAttributeProperty ::= AliasDef | SortingDefinition | DistanceMetricDef // Does not support zero-or-one occurrences
 DistanceMetricDef ::= distance-metric ':' IdentifierWithDashVal
 // Alias
-AliasDef ::= alias IdentifierVal? ':' IdentifierWithDashVal
+AliasDef ::= alias IdentifierVal? ':' IdentifierWithDashVal ('.' IdentifierWithDashVal)*
 // Stemming
 StemmingDefinition ::= stemming ':' IdentifierWithDashVal
 // Rank
 RankDefinition ::= rank ((IdentifierVal? ':' RankingSetting) | ('{' RankingSetting '}'))
-RankingSetting ::= filter | normal
+RankingSetting ::= filter | normal | literal | WordWrapper
 // Indexing Rewrite
 IndexingRewriteState ::= indexing-rewrite ':' none
 // Query Command
-QueryCommandDefinition ::= query-command ':' IdentifierVal | STRING_REG
+QueryCommandDefinition ::= query-command ':' (IdentifierVal | STRING_REG | WordWrapper)
 // Summary
 SummaryDefinition ::= summary ((':' SummaryBodyOptions) | (IdentifierWithDashVal? (type FieldTypeName)? '{' SummaryBody '}'))
                       { mixin="org.intellij.sdk.language.psi.impl.SdSummaryDefinitionMixin" }
@@ -280,29 +284,29 @@ SummaryBodyOptions ::= full | dynamic | (source ':' IdentifierVal (',' Identifie
 // Bolding                       
 BoldingDefinition ::= bolding ':' (on | off | true | false)
 // Index
-IndexDefinition ::= index IdentifierVal (':' IndexProperty) | ('{' IndexProperty* '}')
+IndexDefinition ::= index IdentifierVal? ((':' IndexProperty) | ('{' IndexProperty '}'))
 IndexProperty ::= IndexPropertyOptions*
 private IndexPropertyOptions ::= (alias ':' IdentifierWithDashVal) | StemmingDefinition | (arity ':' INTEGER_REG) | 
                                  (lower-bound ':' INTEGER_REG ('L')?) | (upper-bound ':' INTEGER_REG ('L')?) |
-                                 (dense-posting-list-threshold ':' FLOAT_REG) | enable-bm25 | HnswDefinition
+                                 (dense-posting-list-threshold ':' FLOAT_REG) | enable-bm25 | prefix | HnswDefinition
 HnswDefinition ::= hnsw '{' HnswBody '}'
 HnswBody ::= HnswBodyOptions*
 private HnswBodyOptions ::= (max-links-per-node ':' INTEGER_REG) | (neighbors-to-explore-at-insert ':' INTEGER_REG) |
                             (multi-threaded-indexing ':' (on | off | true | false))
 // Sorting
-SortingDefinition ::= sorting (':' SortingProperty) | ('{' SortingProperty* '}')
+SortingDefinition ::= sorting ((':' SortingProperty) | ('{' SortingProperty* '}'))
 SortingProperty ::= ascending | descending | (function ':' SortingFunction) | (strength ':' SortingStrength) | 
                     (locale ':' IdentifierWithDashVal)
 SortingFunction ::= uca | raw | lowercase
 SortingStrength ::= primary | secondary | tertiary | quaternary | identical
 // Rank Type
-RankTypeDefinition ::= rank-type IdentifierVal ':' IdentifierVal
+RankTypeDefinition ::= rank-type IdentifierVal? ':' IdentifierVal
 // Weighted Set
-WeightedSetDefinition ::= weightedset (':' WeightedSetProperty) | ('{' WeightedSetProperty* '}')  // Does not support 
+WeightedSetDefinition ::= weightedset ((':' WeightedSetProperty) | ('{' WeightedSetProperty* '}'))  // Does not support 
                                                                                                   // zero-or-one occurrences
 WeightedSetProperty ::=  create-if-nonexistent | remove-if-zero
 // Dictionary
-DictionaryDefinition ::= dictionary (':' DictionarySetting) | ('{' DictionarySetting* '}') 
+DictionaryDefinition ::= dictionary ((':' DictionarySetting) | ('{' DictionarySetting* '}')) 
 DictionarySetting ::= hash | btree | cased | uncased
 //***** End of Field's body elements ******//
 
@@ -310,6 +314,8 @@ DictionarySetting ::= hash | btree | cased | uncased
 //---- Util rules -----
 //---------------------
 
+private WordWrapper ::= ID_REG | ID_WITH_DASH_REG | WORD_REG
+
 IdentifierVal ::= KeywordOrIdentifier | ID_REG { mixin="org.intellij.sdk.language.psi.impl.SdIdentifierMixin"
                                                  implements=["org.intellij.sdk.language.psi.SdIdentifier"]
                                                }
@@ -325,8 +331,8 @@ KeywordOrIdentifier ::= schema | search | document | struct | field | type | ind
                         order | ascending | descending | diversity | constants | expression | weight | match |
                         function | macro | inline | text | exact | word | prefix | cased | uncased | substring | suffix | 
                         gram | paged | mutable | alias | sorting | strength | locale | uca | lowercase |
-                        primary | secondary | tertiary | quaternary | identical | rank | filter | normal | none | full | dynamic | 
-                        source | to | strict | loose |
+                        primary | secondary | tertiary | quaternary | identical | rank | filter | normal | literal | 
+                        none | full | dynamic | source | to | strict | loose |
                         bolding | on | off | true | false | id | normalizing | stemming | arity | hnsw | dictionary | hash | btree |
                         fieldset | fields | constant | annotation
                         | attribute | body | header | index | 
@@ -341,5 +347,6 @@ KeywordNotIdentifier ::= struct-field | document-summary | omit-summary-features
                          indexing-rewrite | query-command | matched-elements-only | lower-bound | upper-bound | 
                          dense-posting-list-threshold | enable-bm25 | max-links-per-node | neighbors-to-explore-at-insert | 
                          multi-threaded-indexing | create-if-nonexistent | remove-if-zero | raw-as-base64-in-summary |
-                         onnx-model | cutoff-factor | cutoff-strategy | on-match | on-rank | on-summary
+                         onnx-model | cutoff-factor | cutoff-strategy | on-match | on-rank | on-summary | enable-bit-vectors |
+                         enable-only-bit-vector
                          
\ No newline at end of file