1 files changed, 55 insertions, 7 deletions

diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj
index 255cc3cde70..c9eff88764f 100644
--- a/config-model/src/main/javacc/SchemaParser.jj
+++ b/config-model/src/main/javacc/SchemaParser.jj
@@ -183,11 +183,14 @@ TOKEN :
 | < GRAM_SIZE: "gram-size" >
 | < MAX_LENGTH: "max-length" >
 | < MAX_OCCURRENCES: "max-occurrences" >
+| < MAX_TOKEN_LENGTH: "max-token-length" >
 | < PREFIX: "prefix" >
 | < SUBSTRING: "substring" >
 | < SUFFIX: "suffix" >
 | < CONSTANT: "constant">
 | < ONNX_MODEL: "onnx-model">
+| < SIGNIFICANCE: "significance">
+| < USE_MODEL: "use-model">
 | < INTRAOP_THREADS: "intraop-threads">
 | < INTEROP_THREADS: "interop-threads">
 | < GPU_DEVICE: "gpu-device">
@@ -1366,7 +1369,8 @@ void matchType(ParsedMatchSettings matchInfo) : { }
  */
 void matchItem(ParsedMatchSettings matchInfo) : { }
 {
-    ( matchType(matchInfo) | exactTerminator(matchInfo) | gramSize(matchInfo) | matchSize(matchInfo) | maxTermOccurrences(matchInfo))
+    ( matchType(matchInfo) | exactTerminator(matchInfo) | gramSize(matchInfo) | matchSize(matchInfo) |
+     maxTermOccurrences(matchInfo) | maxTokenLength(matchInfo) )
 }
 
 void exactTerminator(ParsedMatchSettings matchInfo) :
@@ -1411,6 +1415,16 @@ void maxTermOccurrences(ParsedMatchSettings matchInfo) :
     }
 }
 
+void maxTokenLength(ParsedMatchSettings matchInfo) :
+{
+    int maxTokenLength;
+}
+{
+    <MAX_TOKEN_LENGTH> <COLON> maxTokenLength = integer() {
+        matchInfo.setMaxTokenLength(maxTokenLength);
+    }
+}
+
 /**
  * Consumes a rank statement of a field element.
  *
@@ -1742,6 +1756,7 @@ void rankProfileItem(ParsedSchema schema, ParsedRankProfile profile) : { }
       | fieldRankFilter(profile)
       | firstPhase(profile)
       | matchPhase(profile)
+      | diversity(profile)
       | function(profile)
       | mutate(profile)
       | ignoreRankFeatures(profile)
@@ -1761,7 +1776,8 @@ void rankProfileItem(ParsedSchema schema, ParsedRankProfile profile) : { }
       | matchFeatures(profile)
       | summaryFeatures(profile)
       | onnxModelInProfile(profile)
-      | strict(profile) )
+      | strict(profile)
+      | significance(profile))
 }
 
 /**
@@ -1860,14 +1876,14 @@ void matchPhase(ParsedRankProfile profile) :
     MatchPhaseSettings settings = new MatchPhaseSettings();
 }
 {
-    <MATCH_PHASE> lbrace() (matchPhaseItem(settings) (<NL>)*)* <RBRACE>
+    <MATCH_PHASE> lbrace() (matchPhaseItem(profile, settings) (<NL>)*)* <RBRACE>
     {
         settings.checkValid();
-        profile.setMatchPhaseSettings(settings);
+        profile.setMatchPhase(settings);
     }
 }
 
-void matchPhaseItem(MatchPhaseSettings settings) :
+void matchPhaseItem(ParsedRankProfile profile, MatchPhaseSettings settings) :
 {
     String str;
     int num;
@@ -1876,7 +1892,7 @@ void matchPhaseItem(MatchPhaseSettings settings) :
 }
 {
     ( <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); }
-      | diversity(settings)
+      | diversityDeprecated(profile)
       | <ORDER> <COLON> ( <ASCENDING> { settings.setAscending(true); } 
                           | <DESCENDING> { settings.setAscending(false); } )
       | <MAX_HITS> <COLON> num = integer() { settings.setMaxHits(num); }
@@ -1891,7 +1907,18 @@ void matchPhaseItem(MatchPhaseSettings settings) :
  *
  * @param profile The rank profile to modify.
  */
-void diversity(MatchPhaseSettings profile) :
+void diversity(ParsedRankProfile profile) :
+{
+    DiversitySettings settings = new DiversitySettings();
+}
+{
+    <DIVERSITY> lbrace() (diversityItem(settings) (<NL>)*)* <RBRACE>
+    {
+ 	    profile.setDiversity(settings);
+    }
+}
+
+void diversityDeprecated(ParsedRankProfile profile) :
 {
     DiversitySettings settings = new DiversitySettings();
 }
@@ -1899,6 +1926,7 @@ void diversity(MatchPhaseSettings profile) :
     <DIVERSITY> lbrace() (diversityItem(settings) (<NL>)*)* <RBRACE>
     {
  	    profile.setDiversity(settings);
+        deployLogger.logApplicationPackage(Level.WARNING, "'diversity is deprecated inside 'match-phase'. Specify it at 'rank-profile' level.");
     }
 }
 
@@ -1966,10 +1994,12 @@ void secondPhaseItem(ParsedRankProfile profile) :
 {
     String expression;
     int rerankCount;
+    double dropLimit;
 }
 {
     ( expression = expression()                       { profile.setSecondPhaseRanking(expression); }
     | (<RERANK_COUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); }
+    | (<RANK_SCORE_DROP_LIMIT> <COLON> dropLimit = floatValue()) { profile.setSecondPhaseRankScoreDropLimit(dropLimit); }
     )
 }
 
@@ -2115,6 +2145,22 @@ void strict(ParsedRankProfile profile) :
     )
 }
 
+void significance(ParsedRankProfile profile) :
+{}
+{
+    <SIGNIFICANCE> lbrace() (significanceItem(profile) (<NL>)*)* <RBRACE>
+    {}
+}
+
+void significanceItem(ParsedRankProfile profile) :
+{}
+{
+    <USE_MODEL> <COLON> (
+        ( <TRUE>  { profile.setUseSignificanceModel(true);  } ) |
+        ( <FALSE> { profile.setUseSignificanceModel(false); } )
+    )
+}
+
 /**
  * Consumes a match-features block of a rank profile.
  *
@@ -2710,6 +2756,7 @@ String identifierWithDash() :
     | <TARGET_HITS_MAX_ADJUSTMENT_FACTOR>
     | <TERMWISE_LIMIT>
     | <UPPER_BOUND>
+    | <USE_MODEL>
     ) { return token.image; }
 }
 
@@ -2812,6 +2859,7 @@ String identifier() : { }
       | <STEMMING>
       | <STRENGTH>
       | <STRICT>
+      | <SIGNIFICANCE>
       | <STRING>
       | <STRUCT>
       | <SUBSTRING>