Merge pull request #31123 from vespa-engine/toregge/add-max-token-length-to-ilscripts-config

Add max token length to ilscripts config.

35 files changed, 36 insertions, 0 deletions

diff --git a/config-model/src/test/derived/advanced/ilscripts.cfg b/config-model/src/test/derived/advanced/ilscripts.cfg
index 51a49502b64..d633cd97f0c 100644
--- a/config-model/src/test/derived/advanced/ilscripts.cfg
+++ b/config-model/src/test/derived/advanced/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "advanced"
 ilscript[].docfield[] "debug_src"
diff --git a/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg b/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg
index 767c3af3c19..53dc789fbb7 100644
--- a/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg
+++ b/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg
@@ -1,3 +1,4 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "annotationsimplicitstruct"
diff --git a/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg b/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg
index d8e6c882b80..b0a69c5408a 100644
--- a/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg
+++ b/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg
@@ -1,3 +1,4 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "annotationsinheritance"
diff --git a/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg b/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg
index ae4ea621583..5ec1f839429 100644
--- a/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg
+++ b/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg
@@ -1,3 +1,4 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "annotationsinheritance2"
diff --git a/config-model/src/test/derived/annotationsreference/ilscripts.cfg b/config-model/src/test/derived/annotationsreference/ilscripts.cfg
index 812f5e44545..eaa20043be8 100644
--- a/config-model/src/test/derived/annotationsreference/ilscripts.cfg
+++ b/config-model/src/test/derived/annotationsreference/ilscripts.cfg
@@ -1,3 +1,4 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "annotationsreference"
diff --git a/config-model/src/test/derived/annotationssimple/ilscripts.cfg b/config-model/src/test/derived/annotationssimple/ilscripts.cfg
index 9d0962df5be..af179221eb4 100644
--- a/config-model/src/test/derived/annotationssimple/ilscripts.cfg
+++ b/config-model/src/test/derived/annotationssimple/ilscripts.cfg
@@ -1,3 +1,4 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "annotationssimple"
diff --git a/config-model/src/test/derived/arrays/ilscripts.cfg b/config-model/src/test/derived/arrays/ilscripts.cfg
index 98cff642d9e..3f2dae48552 100644
--- a/config-model/src/test/derived/arrays/ilscripts.cfg
+++ b/config-model/src/test/derived/arrays/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "arrays"
 ilscript[].docfield[] "tags"
diff --git a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg
index dec054b33f0..5a3784f7cb9 100644
--- a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg
+++ b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "prefetch"
 ilscript[].docfield[] "singlebyte"
diff --git a/config-model/src/test/derived/attributes/ilscripts.cfg b/config-model/src/test/derived/attributes/ilscripts.cfg
index 6d3ef2799d9..58279759e5f 100644
--- a/config-model/src/test/derived/attributes/ilscripts.cfg
+++ b/config-model/src/test/derived/attributes/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "attributes"
 ilscript[].docfield[] "a1"
diff --git a/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg b/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg
index c20c321ebcf..0b925da4778 100644
--- a/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg
+++ b/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "test"
 ilscript[].docfield[] "str_1"
diff --git a/config-model/src/test/derived/complex/ilscripts.cfg b/config-model/src/test/derived/complex/ilscripts.cfg
index 4405d2fda40..7d025e15703 100644
--- a/config-model/src/test/derived/complex/ilscripts.cfg
+++ b/config-model/src/test/derived/complex/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "complex"
 ilscript[].docfield[] "title"
diff --git a/config-model/src/test/derived/emptydefault/ilscripts.cfg b/config-model/src/test/derived/emptydefault/ilscripts.cfg
index e4242153bce..bbb8e5c556c 100644
--- a/config-model/src/test/derived/emptydefault/ilscripts.cfg
+++ b/config-model/src/test/derived/emptydefault/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "emptydefault"
 ilscript[].docfield[] "one"
diff --git a/config-model/src/test/derived/exactmatch/ilscripts.cfg b/config-model/src/test/derived/exactmatch/ilscripts.cfg
index 21dfbd1371b..1d1bd6d5e8a 100644
--- a/config-model/src/test/derived/exactmatch/ilscripts.cfg
+++ b/config-model/src/test/derived/exactmatch/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "exactmatch"
 ilscript[].docfield[] "tag"
diff --git a/config-model/src/test/derived/hnsw_index/ilscripts.cfg b/config-model/src/test/derived/hnsw_index/ilscripts.cfg
index e48f116f468..c811b93c3df 100644
--- a/config-model/src/test/derived/hnsw_index/ilscripts.cfg
+++ b/config-model/src/test/derived/hnsw_index/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "test"
 ilscript[].docfield[] "t1"
diff --git a/config-model/src/test/derived/id/ilscripts.cfg b/config-model/src/test/derived/id/ilscripts.cfg
index d3ab29f6cd8..121e305059e 100644
--- a/config-model/src/test/derived/id/ilscripts.cfg
+++ b/config-model/src/test/derived/id/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "id"
 ilscript[].docfield[] "uri"
diff --git a/config-model/src/test/derived/indexswitches/ilscripts.cfg b/config-model/src/test/derived/indexswitches/ilscripts.cfg
index 472c1f95cb0..454f675c0a2 100644
--- a/config-model/src/test/derived/indexswitches/ilscripts.cfg
+++ b/config-model/src/test/derived/indexswitches/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "indexswitches"
 ilscript[].docfield[] "title"
diff --git a/config-model/src/test/derived/inheritance/ilscripts.cfg b/config-model/src/test/derived/inheritance/ilscripts.cfg
index d4c804773f0..c966f32a502 100644
--- a/config-model/src/test/derived/inheritance/ilscripts.cfg
+++ b/config-model/src/test/derived/inheritance/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "child"
 ilscript[].docfield[] "onlygrandparent"
diff --git a/config-model/src/test/derived/language/ilscripts.cfg b/config-model/src/test/derived/language/ilscripts.cfg
index 1860f180839..d0abc08f1e0 100644
--- a/config-model/src/test/derived/language/ilscripts.cfg
+++ b/config-model/src/test/derived/language/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "language"
 ilscript[].docfield[] "language"
diff --git a/config-model/src/test/derived/lowercase/ilscripts.cfg b/config-model/src/test/derived/lowercase/ilscripts.cfg
index 8ba4bfa3349..49515e50df4 100644
--- a/config-model/src/test/derived/lowercase/ilscripts.cfg
+++ b/config-model/src/test/derived/lowercase/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "lowercase"
 ilscript[].docfield[] "single_field_source"
diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg
index 0cdf921de25..4a6de4154f8 100644
--- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg
+++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "multiplesummaries"
 ilscript[].docfield[] "a"
diff --git a/config-model/src/test/derived/music/ilscripts.cfg b/config-model/src/test/derived/music/ilscripts.cfg
index f90cdb15baa..f79e8824b69 100644
--- a/config-model/src/test/derived/music/ilscripts.cfg
+++ b/config-model/src/test/derived/music/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "music"
 ilscript[].docfield[] "bgndata"
diff --git a/config-model/src/test/derived/newrank/ilscripts.cfg b/config-model/src/test/derived/newrank/ilscripts.cfg
index b02e09a0496..487d2fca902 100644
--- a/config-model/src/test/derived/newrank/ilscripts.cfg
+++ b/config-model/src/test/derived/newrank/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "newrank"
 ilscript[].docfield[] "bgndata"
diff --git a/config-model/src/test/derived/orderilscripts/ilscripts.cfg b/config-model/src/test/derived/orderilscripts/ilscripts.cfg
index 0ed1589af0a..4918e23efc6 100644
--- a/config-model/src/test/derived/orderilscripts/ilscripts.cfg
+++ b/config-model/src/test/derived/orderilscripts/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "orderilscripts"
 ilscript[].docfield[] "foo"
diff --git a/config-model/src/test/derived/position_array/ilscripts.cfg b/config-model/src/test/derived/position_array/ilscripts.cfg
index ecafbc4a025..3f7611b25d8 100644
--- a/config-model/src/test/derived/position_array/ilscripts.cfg
+++ b/config-model/src/test/derived/position_array/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "position_array"
 ilscript[].docfield[] "pos"
diff --git a/config-model/src/test/derived/position_attribute/ilscripts.cfg b/config-model/src/test/derived/position_attribute/ilscripts.cfg
index d2fc8503ce5..fbd1a293418 100644
--- a/config-model/src/test/derived/position_attribute/ilscripts.cfg
+++ b/config-model/src/test/derived/position_attribute/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "position_attribute"
 ilscript[].docfield[] "pos"
diff --git a/config-model/src/test/derived/position_extra/ilscripts.cfg b/config-model/src/test/derived/position_extra/ilscripts.cfg
index a86dcec92ec..4645798723c 100644
--- a/config-model/src/test/derived/position_extra/ilscripts.cfg
+++ b/config-model/src/test/derived/position_extra/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "position_extra"
 ilscript[].docfield[] "pos_str"
diff --git a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg
index 40c7843a0a4..2d1904cf9d8 100644
--- a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg
+++ b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "prefixexactattribute"
 ilscript[].docfield[] "indexfield0"
diff --git a/config-model/src/test/derived/ranktypes/ilscripts.cfg b/config-model/src/test/derived/ranktypes/ilscripts.cfg
index adcd2f70c70..22526d1aa23 100644
--- a/config-model/src/test/derived/ranktypes/ilscripts.cfg
+++ b/config-model/src/test/derived/ranktypes/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "ranktypes"
 ilscript[].docfield[] "title"
diff --git a/config-model/src/test/derived/schemainheritance/ilscripts.cfg b/config-model/src/test/derived/schemainheritance/ilscripts.cfg
index f7324920fe7..b1ba947f1dc 100644
--- a/config-model/src/test/derived/schemainheritance/ilscripts.cfg
+++ b/config-model/src/test/derived/schemainheritance/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "child"
 ilscript[].docfield[] "pf1"
diff --git a/config-model/src/test/derived/structanyorder/ilscripts.cfg b/config-model/src/test/derived/structanyorder/ilscripts.cfg
index c07f04b3021..a806bc1b712 100644
--- a/config-model/src/test/derived/structanyorder/ilscripts.cfg
+++ b/config-model/src/test/derived/structanyorder/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "annotationsimplicitstruct"
 ilscript[].docfield[] "structfield"
diff --git a/config-model/src/test/derived/tokenization/ilscripts.cfg b/config-model/src/test/derived/tokenization/ilscripts.cfg
index c08b6a54c83..cad8ec81879 100644
--- a/config-model/src/test/derived/tokenization/ilscripts.cfg
+++ b/config-model/src/test/derived/tokenization/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "tokenization"
 ilscript[].docfield[] "text"
diff --git a/config-model/src/test/derived/types/ilscripts.cfg b/config-model/src/test/derived/types/ilscripts.cfg
index 17bed90deb4..73befb221ce 100644
--- a/config-model/src/test/derived/types/ilscripts.cfg
+++ b/config-model/src/test/derived/types/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "types"
 ilscript[].docfield[] "abyte"
diff --git a/config-model/src/test/derived/uri_array/ilscripts.cfg b/config-model/src/test/derived/uri_array/ilscripts.cfg
index 3dd97e5c11f..0dc87b513ce 100644
--- a/config-model/src/test/derived/uri_array/ilscripts.cfg
+++ b/config-model/src/test/derived/uri_array/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "uri_array"
 ilscript[].docfield[] "my_uri"
diff --git a/config-model/src/test/derived/uri_wset/ilscripts.cfg b/config-model/src/test/derived/uri_wset/ilscripts.cfg
index 48e07ef9959..cc45ee5ad8f 100644
--- a/config-model/src/test/derived/uri_wset/ilscripts.cfg
+++ b/config-model/src/test/derived/uri_wset/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 10000
+maxtokenlength 1000
 fieldmatchmaxlength 1000000
 ilscript[].doctype "uri_wset"
 ilscript[].docfield[] "my_uri"
diff --git a/configdefinitions/src/vespa/ilscripts.def b/configdefinitions/src/vespa/ilscripts.def
index acb06abb755..7a286773564 100644
--- a/configdefinitions/src/vespa/ilscripts.def
+++ b/configdefinitions/src/vespa/ilscripts.def
@@ -3,6 +3,8 @@ namespace=vespa.configdefinition
 
 ## The maximum number of occurrences of a given term to index per field
 maxtermoccurrences int default=10000
+## The maximum number of characters for a token
+maxtokenlength int default=1000
 fieldmatchmaxlength int default=1000000
 
 ilscript[].doctype    string