aboutsummaryrefslogtreecommitdiffstats
path: root/indexinglanguage/src/main/javacc/IndexingParser.jj
diff options
context:
space:
mode:
Diffstat (limited to 'indexinglanguage/src/main/javacc/IndexingParser.jj')
-rw-r--r--indexinglanguage/src/main/javacc/IndexingParser.jj14
1 files changed, 10 insertions, 4 deletions
diff --git a/indexinglanguage/src/main/javacc/IndexingParser.jj b/indexinglanguage/src/main/javacc/IndexingParser.jj
index 469d96ead60..29ca5270db8 100644
--- a/indexinglanguage/src/main/javacc/IndexingParser.jj
+++ b/indexinglanguage/src/main/javacc/IndexingParser.jj
@@ -174,6 +174,7 @@ TOKEN :
<LOWER_CASE: "lowercase"> |
<MAX_LENGTH: "max-length"> |
<MAX_OCCURRENCES: "max-occurrences"> |
+ <MAX_TOKEN_LENGTH: "max-token-length"> |
<NGRAM: "ngram"> |
<NORMALIZE: "normalize"> |
<NOW: "now"> |
@@ -407,10 +408,13 @@ Expression embedExp() :
{ return new EmbedExpression(embedders, embedderId, embedderArguments); }
}
-Expression exactExp() : { }
+Expression exactExp() :
{
- ( <EXACT> )
- { return new ExactExpression(); }
+ int maxTokenLength = annotatorCfg.getMaxTokenLength();
+}
+{
+ ( <EXACT> [ <MAX_TOKEN_LENGTH> <COLON> maxTokenLength = integer() ] )
+ { return new ExactExpression(maxTokenLength); }
}
Expression flattenExp() : { }
@@ -686,11 +690,13 @@ AnnotatorConfig tokenizeCfg() :
String str = "SHORTEST";
Integer maxLength;
Integer maxTermOccurrences;
+ Integer maxTokenLength;
}
{
( <STEM> ( <COLON> str = string() ) ? { val.setStemMode(str); } |
- <MAX_LENGTH> <COLON> maxLength = integer() { val.setMaxTokenLength(maxLength); } |
+ <MAX_LENGTH> <COLON> maxLength = integer() { val.setMaxTokenizeLength(maxLength); } |
<MAX_OCCURRENCES> <COLON> maxTermOccurrences = integer() { val.setMaxTermOccurrences(maxTermOccurrences); } |
+ <MAX_TOKEN_LENGTH> <COLON> maxTokenLength = integer() { val.setMaxTokenLength(maxTokenLength); } |
<NORMALIZE> { val.setRemoveAccents(true); } )+
{ return val; }
}