Add linguistics tokens document field writer.

1 files changed, 4 insertions, 0 deletions

diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj
index ae4c3b365d8..a5238afc86a 100644
--- a/config-model/src/main/javacc/SchemaParser.jj
+++ b/config-model/src/main/javacc/SchemaParser.jj
@@ -201,6 +201,7 @@ TOKEN :
 | < FULL: "full" >
 | < STATIC: "static" >
 | < DYNAMIC: "dynamic" >
+| < LINGUISTICS_TOKENS: "linguistics-tokens" >
 | < MATCHED_ELEMENTS_ONLY: "matched-elements-only" >
 | < SSCONTEXTUAL: "contextual" >
 | < SSOVERRIDE: "override" >
@@ -1128,6 +1129,7 @@ void summaryInFieldShort(ParsedField field) :
     <COLON> ( <DYNAMIC> { psf.setDynamic(); }
               | <MATCHED_ELEMENTS_ONLY> { psf.setMatchedElementsOnly(); }
               | (<FULL> | <STATIC>) { psf.setFull(); }
+              | <LINGUISTICS_TOKENS> { psf.setLinguisticsTokens(); }
             )
 }
 
@@ -1173,6 +1175,7 @@ void summaryTransform(ParsedSummaryField field) : { }
     ( <DYNAMIC>              { field.setDynamic(); }
      | <MATCHED_ELEMENTS_ONLY> { field.setMatchedElementsOnly(); }
      | (<FULL> | <STATIC>)   { field.setFull(); }
+     | <LINGUISTICS_TOKENS> { field.setLinguisticsTokens(); }
     )
 }
 
@@ -2712,6 +2715,7 @@ String identifier() : { }
       | <INLINE>
       | <INPUTS>
       | <INTEGER>
+      | <LINGUISTICS_TOKENS>
       | <LITERAL>
       | <LOCALE>
       | <LONG>