Complete grammar, add annotation

6 files changed, 102 insertions, 86 deletions

diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java b/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java
index 4dd2fe8cac1..50eef106b90 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java
+++ b/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java
@@ -10,8 +10,10 @@ import com.intellij.psi.search.FileTypeIndex;
 import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.psi.util.PsiTreeUtil;
 import com.intellij.util.indexing.FileBasedIndex;
+import org.intellij.sdk.language.psi.SdAnnotationFieldDefinition;
 import org.intellij.sdk.language.psi.SdArgumentDefinition;
 import org.intellij.sdk.language.psi.SdDeclaration;
+import org.intellij.sdk.language.psi.SdDocumentAnnotationDefinition;
 import org.intellij.sdk.language.psi.SdDocumentDefinition;
 import org.intellij.sdk.language.psi.SdDocumentFieldDefinition;
 import org.intellij.sdk.language.psi.SdDocumentStructDefinition;
@@ -23,6 +25,7 @@ import org.intellij.sdk.language.psi.SdFunctionDefinition;
 import org.intellij.sdk.language.psi.SdIdentifier;
 import org.intellij.sdk.language.psi.SdImportFieldDefinition;
 import org.intellij.sdk.language.psi.SdRankProfileDefinition;
+import org.intellij.sdk.language.psi.SdSchemaAnnotationDefinition;
 import org.intellij.sdk.language.psi.SdSchemaFieldDefinition;
 import org.intellij.sdk.language.psi.SdSummaryDefinition;
 import org.intellij.sdk.language.psi.SdTypes;
@@ -110,12 +113,12 @@ public class SdUtil {
         if (element.getParent() instanceof SdImportFieldDefinition &&
             element.getNextSibling().getNextSibling().getText().equals("as")) {
             Project project = file.getProject();
-    
+            
             PsiReference docFieldRef = element.getPrevSibling().getPrevSibling().getReference();
             PsiElement docField = docFieldRef != null ? docFieldRef.resolve() : null;
             SdFieldTypeName fieldType = docField != null ? PsiTreeUtil.findChildOfType(docField, SdFieldTypeName.class) : null;
             SdIdentifier docIdentifier = fieldType != null ? PsiTreeUtil.findChildOfType(fieldType, SdIdentifier.class) : null;
-            String docName = docIdentifier != null ? docIdentifier.getName() : null;    
+            String docName = docIdentifier != null ? docIdentifier.getName() : null;
             if (docName == null) {
                 return result;
             }
@@ -165,8 +168,8 @@ public class SdUtil {
                 curRankProfile = getRankProfileParent((SdRankProfileDefinition) curRankProfile);
             }
         }
-
-        for (PsiElement declaration : PsiTreeUtil.collectElements(file, psiElement -> 
+        
+        for (PsiElement declaration : PsiTreeUtil.collectElements(file, psiElement ->
             psiElement instanceof SdDeclaration && !(psiElement instanceof SdArgumentDefinition))) {
             if (name.equals(((SdDeclaration) declaration).getName())) {
                 result.add((SdDeclaration) declaration);
@@ -183,30 +186,33 @@ public class SdUtil {
     
     public static List<PsiElement> findSchemaChildren(PsiElement element) {
         return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, new Class[]{SdDocumentDefinition.class,
-                                                                                   SdSchemaFieldDefinition.class,
-                                                                                   SdImportFieldDefinition.class,
-                                                                                   SdDocumentSummaryDefinition.class,
-                                                                                   SdRankProfileDefinition.class}));
+                                                                                      SdSchemaFieldDefinition.class,
+                                                                                      SdImportFieldDefinition.class,
+                                                                                      SdSchemaAnnotationDefinition.class,
+                                                                                      SdDocumentSummaryDefinition.class,
+                                                                                      SdRankProfileDefinition.class}));
+    }
+    
+    public static List<PsiElement> findAnnotationChildren(PsiElement element) {
+        return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, SdAnnotationFieldDefinition.class));
     }
     
     public static List<PsiElement> findDocumentChildren(PsiElement element) {
-        return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, new Class[]{SdDocumentStructDefinition.class, 
-                                                                                   SdDocumentFieldDefinition.class}));
+        return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, new Class[]{SdDocumentStructDefinition.class,
+                                                                                      SdDocumentAnnotationDefinition.class,
+                                                                                      SdDocumentFieldDefinition.class}));
     }
     
     public static List<PsiElement> findDocumentStructChildren(PsiElement element) {
-        return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element,
-                                                                 SdDocumentStructFieldDefinition.class));
+        return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, SdDocumentStructFieldDefinition.class));
     }
     
     public static List<PsiElement> findRankProfileChildren(PsiElement element) {
-        return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element,
-                                                                 SdFunctionDefinition.class));
+        return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, SdFunctionDefinition.class));
     }
     
     public static List<PsiElement> findDocumentSummaryChildren(PsiElement element) {
-        return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element,
-                                                                 SdSummaryDefinition.class));
+        return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, SdSummaryDefinition.class));
     }
     
     
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf b/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf
index a1bc4c548a0..64cb06f48db 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf
+++ b/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf
@@ -34,19 +34,17 @@
     ]
 }
 
-// IMPORTANT NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats it like zero-to-many)
+// NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats it like zero-to-many)
 
-SdFile ::= Schema
-Schema ::= search IdentifierVal? '{' SchemaBody '}'
+SdFile ::= SchemaDefinition
+SchemaDefinition ::= (search | schema) IdentifierVal? '{' SchemaBody '}'
 SchemaBody ::= SchemaBodyOptions* DocumentDefinition SchemaBodyOptions* // Does not support zero-or-one occurrences
 private SchemaBodyOptions ::= SchemaFieldDefinition | ImportFieldDefinition | DocumentSummaryDefinition | 
                               RankProfileDefinition | 
                               FieldSetDefinition | ConstantDefinition | OnnxModelDefinition | StemmingDefinition |
-                              raw-as-base64-in-summary | AnnotationDefinition
-
-           
-
+                              raw-as-base64-in-summary | SchemaAnnotationDefinition
 
+         
 SchemaFieldDefinition ::= field IdentifierVal type FieldTypeName '{' SchemaFieldBody '}'
                           { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
                             implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
@@ -95,19 +93,26 @@ OnnxModelBody ::= OnnxModelBodyOptions*
 private OnnxModelBodyOptions ::= (file ':' FilePath) | (uri ':' UriPath) | 
                                  ((input | output) (IdentifierVal | STRING) ':' ('.' | '/' | '(' | ')' | IdentifierWithDashVal | WORD_REG)) 
 
-AnnotationDefinition ::= annotation IdentifierVal '{' '}' // todo ask Vespa for syntax
+SchemaAnnotationDefinition ::= AnnotationDefinition
+                        { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
+                          implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+                          methods=[getName setName getType getTypeName getNameIdentifier getPresentation] }
+                          
+private AnnotationDefinition ::= annotation IdentifierVal '{' AnnotationFieldDefinition* '}'
+AnnotationFieldDefinition ::= field IdentifierVal type FieldTypeName '{' '}'
+                          { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
+                            implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+                            methods=[getName setName getType getTypeName getNameIdentifier getPresentation] }
 
 //-------------------------
 //--- Expressions rules ---
 //-------------------------
 RankingExpression ::= ParenthesisedExpr | BooleanExpr |ArithmeticExpr | IfFunctionExpr | 
-                      QueryDefinitionExpr | FunctionCallExpr | PrimitiveExpr //ReduceExpr | 
+                      QueryDefinitionExpr | FunctionCallExpr | PrimitiveExpr
 
 IfFunctionExpr ::= "if" '(' (InListRankingExpression | RankingExpression) ',' RankingExpression ',' RankingExpression ')'
 InListRankingExpression ::= RankingExpression "in" '[' RankingExpression (',' RankingExpression)* ']'
 
-//ReduceExpr ::= reduce '(' RankingExpression ',' (avg|count|max|median|min|prod|sum) (',' RankingExpression)* ')'
-
 BooleanExpr ::= RankingExpression COMPARISON_OPERATOR RankingExpression
                 
 ArithmeticExpr ::= RankingExpression ARITHMETIC_OPERATOR RankingExpression
@@ -132,14 +137,15 @@ private RankProfileBodyOptions ::= MatchPhaseDefinition | NumThreadsDefinition |
                            ignore-default-rank-features | RankPropertiesDefinition | FirstPhaseDefinition | 
                            SummaryFeaturesDefinition | RankFeaturesDefinition | SecondPhaseDefinition | ConstantsDefinition |
                            RankDefinition | RankTypeDefinition | MinHitsDefinition | NumSearchPartitionDefinition
-                           // | FieldWeightDefinition | ExecuteDefinition | RankDegradationDefinition // todo check with Vespa if need to add these
 
 MatchPhaseDefinition ::= match-phase '{' MatchPhaseBody '}'
-MatchPhaseBody ::= MatchPhaseBodyOptions+ // todo check with Vespa- are there more options?
+MatchPhaseBody ::= MatchPhaseBodyOptions+
 MatchPhaseBodyOptions ::= (attribute ':' IdentifierVal (order ':' (ascending | descending))?) | (max-hits ':' INTEGER_REG)
                           | DiversityDefinition // Does not support zero-or-one occurrences
-DiversityDefinition ::= diversity '{' (attribute ':' IdentifierVal min-groups ':' INTEGER_REG) |
-                                      (min-groups ':' INTEGER_REG attribute ':' IdentifierVal) '}'
+DiversityDefinition ::= diversity '{' DiversityBody '}'
+DiversityBody ::= DiversityBodyOptions*
+private DiversityBodyOptions ::= (attribute ':' IdentifierVal) | (min-groups ':' INTEGER_REG) | (cutoff-factor ':' FLOAT_REG) |
+                                 (cutoff-strategy ':' (strict | loose))
 
 private NumThreadsDefinition ::= num-threads-per-search ':' INTEGER_REG
 private TermwiseLimitDefinition ::= termwise-limit ':' (FLOAT_REG | INTEGER_REG)
@@ -177,9 +183,7 @@ RankFeaturesDefinition ::= rank-features (':' RankFeature+) | ('{' RankFeature*
 
 ConstantsDefinition ::= constants '{' (IdentifierVal ':' RankPropertiesValue)* '}'
 
-//******** Rank features *********
 RankFeature ::= QueryDefinition | ItemRawScoreDefinition | FunctionCallExpr | (IdentifierWithDashVal ('.' IdentifierWithDashVal)* )
-
 QueryDefinition ::= "query" '(' IdentifierWithDashVal ')'
                     { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
                       implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
@@ -188,30 +192,7 @@ ItemRawScoreDefinition ::=  "itemRawScore" '(' IdentifierVal ')'
                             { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
                              implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
                              methods=[getName setName getType getTypeName getNameIdentifier getPresentation] }
-                             
-//                QueryFeature | DocumentFeature | QueryTermFieldMatchFeature | AttributeMatchFeature | RankScoreFeature |
-//                GlobalFeature | MatchOperatorScoreFeature | UtilityFeature | RankingExpressionMethod 
-                //todo add the rest (| FieldMatchFeature | QueryFieldSimilarityFeature | IdxMultiStrFieldFeature | MultiFieldsAttributesFeature)
-                
-// todo maybe not specify all of the rank features here? Maybe it would be better to write it more generally? like ID '(' ID ')' '.' ID.. so it could handle new features in the future
-//QueryFeature ::= QueryDefinition | (term '(' (FLOAT_REG | INTEGER_REG) ')' '.' (significant | weight | connectedness)) // todo add queryTermCount 
-//DocumentFeature ::= (fieldLength '(' IdentifierVal ')') | (attribute '(' IdentifierVal ')' ('.' count)?) |
-//                    (attribute '(' IdentifierVal ',' (INTEGER_REG | IdentifierVal) ')' ('.' (weight | contains))?) |
-//                    (tensorFromWeightedSet '(' ((attribute'('IdentifierVal')') | QueryDefinition | IdentifierVal) ',' IdentifierVal ')') |
-//                    (tensorFromLabels '(' ((attribute'('IdentifierVal')') | QueryDefinition | IdentifierVal) ',' IdentifierVal ')')
-//QueryTermFieldMatchFeature ::= (matchCount'('IdentifierVal')') | (matches'('IdentifierVal (',' INTEGER_REG)? ')') // todo add the rest
-//AttributeMatchFeature ::= (attributeMatch'('IdentifierVal')' ('.' (completeness | queryCompleteness | fieldCompleteness | 
-//                           normalizedWeight | normalizedWeightedWeight | matches | totalWeight | averageWeight | maxWeight))?) |
-//                          (distance'('IdentifierVal')' ('.' (index | latitude | longitude))?) |
-//                          (distanceToPath'('IdentifierVal')' ('.' (distance | traveled | product))?) | (age'('IdentifierVal')')  // todo add closeness and freshness
-//RankScoreFeature ::= (nativeDotProduct'('IdentifierVal')') // todo add the rest
-//GlobalFeature ::= (random'('IdentifierVal')' '.' match) | (random) // todo add the rest
-//MatchOperatorScoreFeature ::= (rawScore'('IdentifierVal')') | ItemRawScoreDefinition
-//UtilityFeature ::= (dotProduct'('IdentifierVal ',' IdentifierVal ')')  // todo add the rest
-//RankingExpressionMethod ::= "rankingExpression" '(' (RankFeature | IdentifierVal) ')'
-//***** End of Rank features *****
-
-
+                            
 //-------------------------
 //---- Document rules -----
 //-------------------------
@@ -221,7 +202,12 @@ DocumentDefinition ::= document (IdentifierVal (inherits IdentifierVal (',' Iden
                         methods=[getName setName getType getTypeName getNameIdentifier getPresentation] }
 
 DocumentBody ::= DocumentBodyOptions*
-DocumentBodyOptions ::= DocumentStructDefinition | DocumentFieldDefinition
+DocumentBodyOptions ::= DocumentStructDefinition | DocumentFieldDefinition | DocumentAnnotationDefinition
+
+DocumentAnnotationDefinition ::= AnnotationDefinition
+                        { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
+                          implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+                          methods=[getName setName getType getTypeName getNameIdentifier getPresentation] }
 
 DocumentStructDefinition ::= struct IdentifierVal '{' DocumentStructBody '}'
                              { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
@@ -246,7 +232,7 @@ private DocumentFieldBodyOptions ::= StructFieldDefinition | MatchDefinition | I
                              AliasDef | RankDefinition | IndexingRewriteState | QueryCommandDefinition | SummaryDefinition |
                              BoldingDefinition | (id ':' INTEGER_REG) | IndexDefinition | (normalizing ':' IdentifierWithDashVal) | 
                              SortingDefinition | StemmingDefinition | (weight ': INTEGER_REG') | WeightedSetDefinition |
-                             RankTypeDefinition | DictionaryDefinition // todo check with Vespa- SummaryToDefinition is needed here? it's deprecated
+                             RankTypeDefinition | DictionaryDefinition
 
 //***** Field's body elements ******//
 // Struct
@@ -265,7 +251,7 @@ MatchProperty ::= text | exact | exact-terminator | word | prefix | cased | unca
 // Indexing
 IndexingDefinition ::= indexing (':' IndexingStatement) | ('{' IndexingStatement+ '}')
 IndexingStatement ::= IndexingStatementOptions (('|' IndexingStatementOptions)*) | ((';' IndexingStatementOptions)*)
-                      // Does not support zero-or-one occurrences // todo check with Vespa- Can "input" be here?
+                      // Does not support zero-or-one occurrences
 IndexingStatementOptions ::= summary | attribute | index | "set_language"
 // Attribute
 AttributeDefinition ::= attribute ((':' SimpleAttributeProperty) | ('{' (SimpleAttributeProperty | ComplexAttributeProperty)+ '}'))
@@ -278,7 +264,7 @@ AliasDef ::= alias IdentifierVal? ':' IdentifierWithDashVal
 StemmingDefinition ::= stemming ':' IdentifierWithDashVal
 // Rank
 RankDefinition ::= rank ((IdentifierVal? ':' RankingSetting) | ('{' RankingSetting '}'))
-RankingSetting ::= filter | normal | literal // todo check with Vespa- is "literal" good here?
+RankingSetting ::= filter | normal
 // Indexing Rewrite
 IndexingRewriteState ::= indexing-rewrite ':' none
 // Query Command
@@ -330,13 +316,13 @@ IdentifierWithDashVal ::= ID_WITH_DASH_REG | IdentifierVal { implements=["org.in
                          
 // Those lists of keywords (KeywordOrIdentifier and KeywordNotIdentifier) have to be synchronized with sd.flex file.
 // If you add a keyword here, you should add it to the sd.flex file as well.
-KeywordOrIdentifier ::= search | document | struct | field | type | indexing | input | output | inherits | import | as |
-                        raw | uri | file | annotationreference | array | weightedset | map |
-                        order | ascending | descending | diversity | constants | literal | expression | weight | match |
+KeywordOrIdentifier ::= schema | search | document | struct | field | type | indexing | input | output | inherits | 
+                        import | as | raw | uri | file | annotationreference | array | weightedset | map |
+                        order | ascending | descending | diversity | constants | expression | weight | match |
                         function | macro | inline | text | exact | word | prefix | cased | uncased | substring | suffix | 
                         gram | paged | mutable | alias | sorting | strength | locale | uca | lowercase |
                         primary | secondary | tertiary | quaternary | identical | rank | filter | normal | none | full | dynamic | 
-                        source | to |
+                        source | to | strict | loose | //execute |
                         bolding | on | off | true | false | id | normalizing | stemming | arity | hnsw | dictionary | hash | btree |
                         fieldset | fields | constant | annotation
                         // KeywordNotIdentifier ::= (not enforced in this version)
@@ -351,5 +337,5 @@ KeywordNotIdentifier ::= struct-field | document-summary | omit-summary-features
                          indexing-rewrite | query-command | matched-elements-only | lower-bound | upper-bound | 
                          dense-posting-list-threshold | enable-bm25 | max-links-per-node | neighbors-to-explore-at-insert | 
                          multi-threaded-indexing | create-if-nonexistent | remove-if-zero | raw-as-base64-in-summary |
-                         onnx-model
+                         onnx-model | cutoff-factor | cutoff-strategy | on-match | on-rank | on-summary
                          
\ No newline at end of file
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java
index f832a0d01e2..05b38d981c3 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java
+++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java
@@ -3,9 +3,11 @@ package org.intellij.sdk.language.psi;
 public enum SdDeclarationType {
     DOCUMENT("Document"),
     STRUCT("Struct"),
+    ANNOTATION("Annotation"),
     SCHEMA_FIELD("Field (in Schema)"),
     DOCUMENT_FIELD("Field (in Document)"),
     STRUCT_FIELD("Struct-Field"),
+    ANNOTATION_FIELD("Field (in Annotation)"),
     DOCUMENT_STRUCT_FIELD("Field (in Struct)"),
     IMPORTED_FIELD("Imported Field"),
     DOCUMENT_SUMMARY("Document-Summary"),
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdPsiImplUtil.java b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdPsiImplUtil.java
index d22caa5adaa..0645bd3c51f 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdPsiImplUtil.java
+++ b/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdPsiImplUtil.java
@@ -5,16 +5,16 @@ import com.intellij.lang.ASTNode;
 import com.intellij.navigation.ItemPresentation;
 import com.intellij.openapi.util.TextRange;
 import com.intellij.psi.PsiElement;
-import com.intellij.psi.PsiFile;
 import com.intellij.psi.PsiReference;
 import com.intellij.psi.util.PsiTreeUtil;
 import org.intellij.sdk.language.SdIcons;
 import org.intellij.sdk.language.SdReference;
 import org.intellij.sdk.language.SdUtil;
-import org.intellij.sdk.language.hierarchy.SdHierarchyUtil;
+import org.intellij.sdk.language.psi.SdAnnotationFieldDefinition;
 import org.intellij.sdk.language.psi.SdArgumentDefinition;
 import org.intellij.sdk.language.psi.SdDeclaration;
 import org.intellij.sdk.language.psi.SdDeclarationType;
+import org.intellij.sdk.language.psi.SdDocumentAnnotationDefinition;
 import org.intellij.sdk.language.psi.SdDocumentDefinition;
 import org.intellij.sdk.language.psi.SdDocumentFieldDefinition;
 import org.intellij.sdk.language.psi.SdDocumentStructDefinition;
@@ -31,6 +31,7 @@ import org.intellij.sdk.language.psi.SdImportFieldDefinition;
 import org.intellij.sdk.language.psi.SdItemRawScoreDefinition;
 import org.intellij.sdk.language.psi.SdQueryDefinition;
 import org.intellij.sdk.language.psi.SdRankProfileDefinition;
+import org.intellij.sdk.language.psi.SdSchemaAnnotationDefinition;
 import org.intellij.sdk.language.psi.SdSchemaFieldDefinition;
 import org.intellij.sdk.language.psi.SdStructFieldDefinition;
 import org.intellij.sdk.language.psi.SdSummaryDefinition;
@@ -41,7 +42,7 @@ import org.jetbrains.annotations.Nullable;
 import javax.swing.Icon;
 
 /**
- * I this class there are implementations of methods from rules in the .bnf file. While generating the psi files
+ * In this class there are implementations of methods from rules in the .bnf file. While generating the psi files
  * (classes and interfaces) the implementations would be taken from here.
  */
 public class SdPsiImplUtil {
@@ -68,12 +69,17 @@ public class SdPsiImplUtil {
             return SdDeclarationType.DOCUMENT;
         } else if (declaration instanceof SdDocumentStructDefinition) {
             return SdDeclarationType.STRUCT;
+        } else if (declaration instanceof SdSchemaAnnotationDefinition ||
+                   declaration instanceof SdDocumentAnnotationDefinition) {
+            return SdDeclarationType.ANNOTATION;
         } else if (declaration instanceof SdDocumentStructFieldDefinition) {
             return SdDeclarationType.DOCUMENT_STRUCT_FIELD;
         } else if (declaration instanceof SdDocumentFieldDefinition) {
             return SdDeclarationType.DOCUMENT_FIELD;
         } else if (declaration instanceof SdStructFieldDefinition) {
             return SdDeclarationType.STRUCT_FIELD;
+        } else if (declaration instanceof SdAnnotationFieldDefinition) {
+            return SdDeclarationType.ANNOTATION_FIELD;
         } else if (declaration instanceof SdQueryDefinition) {
             return SdDeclarationType.QUERY;
         } else if (declaration instanceof SdItemRawScoreDefinition) {
@@ -119,7 +125,7 @@ public class SdPsiImplUtil {
     // ################################### //
     // ##### getName implementations ##### //
     // ################################### //
-
+    
     @NotNull
     public static String getName(SdIdentifier element) {
         if (element != null) {
@@ -136,7 +142,7 @@ public class SdPsiImplUtil {
         if (declaration instanceof SdImportFieldDefinition) {
             ASTNode asNode = declaration.getNode().findChildByType(SdTypes.AS);
             node = declaration.getNode().findChildByType(SdTypes.IDENTIFIER_VAL, asNode);
-        } else if (declaration instanceof SdRankProfileDefinition || declaration instanceof SdDocumentSummaryDefinition 
+        } else if (declaration instanceof SdRankProfileDefinition || declaration instanceof SdDocumentSummaryDefinition
                    || declaration instanceof SdQueryDefinition) {
             node = declaration.getNode().findChildByType(SdTypes.IDENTIFIER_WITH_DASH_VAL);
         } else {
@@ -253,7 +259,7 @@ public class SdPsiImplUtil {
             public String getPresentableText() {
                 if (element instanceof SdFunctionDefinition) {
                     return SdUtil.createFunctionDescription((SdFunctionDefinition) element);
-                } 
+                }
                 SdRankProfileDefinition rankProfileParent = PsiTreeUtil.getParentOfType(element, SdRankProfileDefinition.class);
                 if (rankProfileParent != null) {
                     if (element instanceof SdQueryDefinition || element instanceof SdItemRawScoreDefinition) {
@@ -276,7 +282,8 @@ public class SdPsiImplUtil {
                 if (element instanceof SdFile) {
                     return SdIcons.FILE;
                 } else if (element instanceof SdSchemaFieldDefinition || element instanceof SdDocumentFieldDefinition ||
-                           element instanceof SdQueryDefinition || element instanceof SdItemRawScoreDefinition) {
+                           element instanceof SdAnnotationFieldDefinition || element instanceof SdQueryDefinition ||
+                           element instanceof SdItemRawScoreDefinition) {
                     return AllIcons.Nodes.Field;
                 } else if (element instanceof SdStructFieldDefinition  ||
                            element instanceof SdDocumentStructFieldDefinition) {
@@ -295,6 +302,9 @@ public class SdPsiImplUtil {
                     return SdIcons.DOCUMENT_SUMMARY;
                 } else if (element instanceof SdDocumentDefinition) {
                     return SdIcons.DOCUMENT;
+                } else if (element instanceof SdSchemaAnnotationDefinition ||
+                           element instanceof SdDocumentAnnotationDefinition) {
+                    return AllIcons.Nodes.ObjectTypeAttribute;
                 }
                 else {
                     return null;
@@ -305,7 +315,7 @@ public class SdPsiImplUtil {
     
     public static ItemPresentation getPresentation(final SdSummaryDefinition element) {
         return new ItemPresentation() {
-
+            
             @Override
             public String getPresentableText() {
                 return element.getName();
@@ -326,7 +336,7 @@ public class SdPsiImplUtil {
     
     public static ItemPresentation getPresentation(final SdFirstPhaseDefinition element) {
         return new ItemPresentation() {
-
+            
             @Override
             public String getPresentableText() {
                 SdRankProfileDefinition rankProfile = PsiTreeUtil.getParentOfType(element, SdRankProfileDefinition.class);
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/sd.flex b/sd-plugin/src/main/java/org/intellij/sdk/language/sd.flex
index c657af4f658..5150250665e 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/sd.flex
+++ b/sd-plugin/src/main/java/org/intellij/sdk/language/sd.flex
@@ -42,6 +42,7 @@ WORD = \w+
   // These tokens are Parsed uses these return values to match token squence to a parser rule.
   
   "search"                   { return SEARCH; }
+  "schema"                   { return SCHEMA; }
   "document"                 { return DOCUMENT; }
   "inherits"                 { return INHERITS; }
   "struct"                   { return STRUCT; }
@@ -118,6 +119,10 @@ WORD = \w+
   "max-hits"                 { return MAX_HITS; }
   "diversity"                { return DIVERSITY; }
   "min-groups"               { return MIN_GROUPS; }
+  "cutoff-factor"            { return CUTOFF_FACTOR; }
+  "cutoff-strategy"          { return CUTOFF_STRATEGY; }
+  "loose"                    { return LOOSE; }
+  "strict"                   { return STRICT; }
   "rank-properties"          { return RANK_PROPERTIES; }
   
   "first-phase"              { return FIRST_PHASE; }
@@ -132,7 +137,6 @@ WORD = \w+
   "min-hits-per-thread"      { return MIN_HITS_PER_THREAD; }
   "num-search-partition"     { return NUM_SEARCH_PARTITION; }
   "constants"                { return CONSTANTS; }
-  "literal"                  { return LITERAL; }
   "second-phase"             { return SECOND_PHASE; }
   "rerank-count"             { return RERANK_COUNT; }
 
@@ -170,6 +174,9 @@ WORD = \w+
   "rank-type"                { return RANK_TYPE; }
   "onnx-model"               { return ONNX_MODEL; }
   "raw-as-base64-in-summary" { return RAW_AS_BASE64_IN_SUMMARY; }
+  "on-match"                 { return ON_MATCH; }
+  "on-rank"                  { return ON_RANK; }
+  "on-summary"               { return ON_SUMMARY; }
 
   "function"                 { return FUNCTION; }
   "macro"                    { return MACRO; }
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java b/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java
index a5f45e6b26b..ac7b79c8dd3 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java
+++ b/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java
@@ -8,61 +8,63 @@ import com.intellij.navigation.ItemPresentation;
 import com.intellij.psi.NavigatablePsiElement;
 import com.intellij.psi.PsiElement;
 import org.intellij.sdk.language.SdUtil;
+import org.intellij.sdk.language.psi.SdDocumentAnnotationDefinition;
 import org.intellij.sdk.language.psi.SdDocumentDefinition;
 import org.intellij.sdk.language.psi.SdDocumentStructDefinition;
 import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition;
 import org.intellij.sdk.language.psi.SdFile;
 import org.intellij.sdk.language.psi.SdRankProfileDefinition;
+import org.intellij.sdk.language.psi.SdSchemaAnnotationDefinition;
 import org.jetbrains.annotations.NotNull;
 
 import java.util.ArrayList;
 import java.util.List;
 
 public class SdStructureViewElement implements StructureViewTreeElement, SortableTreeElement {
-
+    
     private final NavigatablePsiElement myElement;
-
+    
     public SdStructureViewElement(NavigatablePsiElement element) {
         this.myElement = element;
     }
-
+    
     @Override
     public Object getValue() {
         return myElement;
     }
-
+    
     @Override
     public void navigate(boolean requestFocus) {
         myElement.navigate(requestFocus);
     }
-
+    
     @Override
     public boolean canNavigate() {
         return myElement.canNavigate();
     }
-
+    
     @Override
     public boolean canNavigateToSource() {
         return myElement.canNavigateToSource();
     }
-
+    
     @NotNull
     @Override
     public String getAlphaSortKey() {
         String name = myElement.getName();
         return name != null ? name : "";
     }
-
+    
     @NotNull
     @Override
     public ItemPresentation getPresentation() {
         ItemPresentation presentation = myElement.getPresentation();
         return presentation != null ? presentation : new PresentationData();
     }
-
+    
     @Override
     public TreeElement @NotNull [] getChildren() {
-        List<PsiElement> children = new ArrayList<>();;
+        List<PsiElement> children = new ArrayList<>();
         if (myElement instanceof SdFile) {
             children = SdUtil.findSchemaChildren(myElement);
         } else if (myElement instanceof SdDocumentDefinition) {
@@ -73,6 +75,9 @@ public class SdStructureViewElement implements StructureViewTreeElement, Sortabl
             children = SdUtil.findRankProfileChildren(myElement);
         } else if (myElement instanceof SdDocumentSummaryDefinition) {
             children = SdUtil.findDocumentSummaryChildren(myElement);
+        } else if (myElement instanceof SdSchemaAnnotationDefinition ||
+                   myElement instanceof SdDocumentAnnotationDefinition) {
+            children = SdUtil.findAnnotationChildren(myElement);
         }
         
         List<TreeElement> treeElements = new ArrayList<>(children.size());