summaryrefslogtreecommitdiffstats
path: root/indexinglanguage
diff options
context:
space:
mode:
authorTor Egge <Tor.Egge@online.no>2024-05-06 12:37:44 +0200
committerTor Egge <Tor.Egge@online.no>2024-05-06 12:37:44 +0200
commit2a08cdf582e5bbd361863e8e7bf27048e8b00389 (patch)
tree589405cad83be6d6c2e5234e9aa077d986175789 /indexinglanguage
parent11252dce21f47a51570f24c6434ea99f9a0c11f8 (diff)
Rename max token length to max tokenize length in linguistics annotator
config.
Diffstat (limited to 'indexinglanguage')
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java2
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java4
-rw-r--r--indexinglanguage/src/main/javacc/IndexingParser.jj2
-rw-r--r--indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java2
4 files changed, 5 insertions, 5 deletions
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java
index b807ad4cb65..849bc075a64 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java
@@ -66,7 +66,7 @@ public final class TokenizeExpression extends Expression {
if (config.getStemMode() != StemMode.NONE) {
ret.append(" stem:\""+config.getStemMode()+"\"");
}
- if (config.hasNonDefaultMaxTokenLength()) {
+ if (config.hasNonDefaultMaxTokenizeLength()) {
ret.append(" max-length:" + config.getMaxTokenizeLength());
}
if (config.hasNonDefaultMaxTermOccurrences()) {
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java
index 7b6f350d831..4e5ef0d90df 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java
@@ -82,7 +82,7 @@ public class AnnotatorConfig implements Cloneable {
return this;
}
- public AnnotatorConfig setMaxTokenLength(int maxTokenizeLength) {
+ public AnnotatorConfig setMaxTokenizeLength(int maxTokenizeLength) {
this.maxTokenizeLength = maxTokenizeLength;
return this;
}
@@ -91,7 +91,7 @@ public class AnnotatorConfig implements Cloneable {
return maxTokenizeLength;
}
- public boolean hasNonDefaultMaxTokenLength() {
+ public boolean hasNonDefaultMaxTokenizeLength() {
return maxTokenizeLength != DEFAULT_MAX_TOKENIZE_LENGTH;
}
diff --git a/indexinglanguage/src/main/javacc/IndexingParser.jj b/indexinglanguage/src/main/javacc/IndexingParser.jj
index 469d96ead60..77591d3e54e 100644
--- a/indexinglanguage/src/main/javacc/IndexingParser.jj
+++ b/indexinglanguage/src/main/javacc/IndexingParser.jj
@@ -689,7 +689,7 @@ AnnotatorConfig tokenizeCfg() :
}
{
( <STEM> ( <COLON> str = string() ) ? { val.setStemMode(str); } |
- <MAX_LENGTH> <COLON> maxLength = integer() { val.setMaxTokenLength(maxLength); } |
+ <MAX_LENGTH> <COLON> maxLength = integer() { val.setMaxTokenizeLength(maxLength); } |
<MAX_OCCURRENCES> <COLON> maxTermOccurrences = integer() { val.setMaxTermOccurrences(maxTermOccurrences); } |
<NORMALIZE> { val.setRemoveAccents(true); } )+
{ return val; }
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java
index 136e71564d8..461c915acef 100644
--- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java
@@ -194,7 +194,7 @@ public class LinguisticsAnnotatorTestCase {
Linguistics linguistics = new SimpleLinguistics();
- LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, new AnnotatorConfig().setMaxTokenLength(12));
+ LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, new AnnotatorConfig().setMaxTokenizeLength(12));
assertTrue(annotator.annotate(shortValue));
assertEquals(spanTree, shortValue.getSpanTree(SpanTrees.LINGUISTICS));