Merge pull request #31119 from vespa-engine/toregge/rename-max-token-length-to-max-tokenize-length-in-linguistics-annotator-config

Rename max token length to max tokenize length in linguistics annotator

6 files changed, 7 insertions, 7 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java
index e29f683761f..1ff019038fc 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java
@@ -64,7 +64,7 @@ public class TextMatch extends Processor {
         if (fieldMatching != null) {
             var maxLength = fieldMatching.maxLength();
             if (maxLength != null) {
-                ret.setMaxTokenLength(maxLength);
+                ret.setMaxTokenizeLength(maxLength);
             }
             var maxTermOccurrences = fieldMatching.maxTermOccurrences();
             if (maxTermOccurrences != null) {
diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java
index 86b0a2e78ad..3088083912b 100644
--- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java
+++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java
@@ -72,7 +72,7 @@ public class ScriptManager {
         Map<String, Map<String, DocumentScript>> documentFieldScripts = new HashMap<>(config.ilscript().size());
         ScriptParserContext parserContext = new ScriptParserContext(linguistics, embedders);
         parserContext.getAnnotatorConfig().setMaxTermOccurrences(config.maxtermoccurrences());
-        parserContext.getAnnotatorConfig().setMaxTokenLength(config.fieldmatchmaxlength());
+        parserContext.getAnnotatorConfig().setMaxTokenizeLength(config.fieldmatchmaxlength());
 
         for (IlscriptsConfig.Ilscript ilscript : config.ilscript()) {
             DocumentType documentType = docTypeMgr.getDocumentType(ilscript.doctype());
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java
index b807ad4cb65..849bc075a64 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java
@@ -66,7 +66,7 @@ public final class TokenizeExpression extends Expression {
         if (config.getStemMode() != StemMode.NONE) {
             ret.append(" stem:\""+config.getStemMode()+"\"");
         }
-        if (config.hasNonDefaultMaxTokenLength()) {
+        if (config.hasNonDefaultMaxTokenizeLength()) {
             ret.append(" max-length:" + config.getMaxTokenizeLength());
         }
         if (config.hasNonDefaultMaxTermOccurrences()) {
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java
index 7b6f350d831..4e5ef0d90df 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java
@@ -82,7 +82,7 @@ public class AnnotatorConfig implements Cloneable {
         return this;
     }
 
-    public AnnotatorConfig setMaxTokenLength(int maxTokenizeLength) {
+    public AnnotatorConfig setMaxTokenizeLength(int maxTokenizeLength) {
         this.maxTokenizeLength = maxTokenizeLength;
         return this;
     }
@@ -91,7 +91,7 @@ public class AnnotatorConfig implements Cloneable {
         return maxTokenizeLength;
     }
 
-    public boolean hasNonDefaultMaxTokenLength() {
+    public boolean hasNonDefaultMaxTokenizeLength() {
         return maxTokenizeLength != DEFAULT_MAX_TOKENIZE_LENGTH;
     }
 
diff --git a/indexinglanguage/src/main/javacc/IndexingParser.jj b/indexinglanguage/src/main/javacc/IndexingParser.jj
index 469d96ead60..77591d3e54e 100644
--- a/indexinglanguage/src/main/javacc/IndexingParser.jj
+++ b/indexinglanguage/src/main/javacc/IndexingParser.jj
@@ -689,7 +689,7 @@ AnnotatorConfig tokenizeCfg() :
 }
 {
     ( <STEM> ( <COLON> str = string() ) ? { val.setStemMode(str); } |
-      <MAX_LENGTH> <COLON>  maxLength = integer() { val.setMaxTokenLength(maxLength); } |
+      <MAX_LENGTH> <COLON>  maxLength = integer() { val.setMaxTokenizeLength(maxLength); } |
       <MAX_OCCURRENCES> <COLON>  maxTermOccurrences = integer() { val.setMaxTermOccurrences(maxTermOccurrences); } |
       <NORMALIZE> { val.setRemoveAccents(true); } )+
     { return val; }
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java
index 136e71564d8..461c915acef 100644
--- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java
@@ -194,7 +194,7 @@ public class LinguisticsAnnotatorTestCase {
 
         Linguistics linguistics = new SimpleLinguistics();
 
-        LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, new AnnotatorConfig().setMaxTokenLength(12));
+        LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, new AnnotatorConfig().setMaxTokenizeLength(12));
 
         assertTrue(annotator.annotate(shortValue));
         assertEquals(spanTree, shortValue.getSpanTree(SpanTrees.LINGUISTICS));