Let the default field match max length be controlable by config override as for maxtermoccurences.

30 files changed, 61 insertions, 30 deletions

diff --git a/config-model/src/test/derived/advanced/ilscripts.cfg b/config-model/src/test/derived/advanced/ilscripts.cfg
index ffa9a7e060f..197b47ccdc4 100644
--- a/config-model/src/test/derived/advanced/ilscripts.cfg
+++ b/config-model/src/test/derived/advanced/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "advanced"
 ilscript[0].docfield[0] "debug_src"
 ilscript[0].docfield[1] "attributes_src"
@@ -21,4 +22,4 @@ ilscript[0].content[10] "input debug_src | passthrough debug_src"
 ilscript[0].content[11] "input product2_src | passthrough product2_src"
 ilscript[0].content[12] "input product3_src | passthrough product3_src"
 ilscript[0].content[13] "input product_src | passthrough product_src"
-ilscript[0].content[14] "input title_src | passthrough title_src"
\ No newline at end of file
+ilscript[0].content[14] "input title_src | passthrough title_src"
diff --git a/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg b/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg
index b49e52092c2..82965f82f1a 100644
--- a/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg
+++ b/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg
@@ -1,2 +1,3 @@
 maxtermoccurrences 100
-ilscript[0].doctype "annotationsimplicitstruct"
\ No newline at end of file
+fieldmatchmaxlength 1000000
+ilscript[0].doctype "annotationsimplicitstruct"
diff --git a/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg b/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg
index a6b1aabfb47..a1a2e216947 100755..100644
--- a/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg
+++ b/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg
@@ -1,2 +1,3 @@
 maxtermoccurrences 100
-ilscript[0].doctype "annotationsinheritance"
\ No newline at end of file
+fieldmatchmaxlength 1000000
+ilscript[0].doctype "annotationsinheritance"
diff --git a/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg b/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg
index 2717f4ab7e5..d5fc9dae6f6 100755..100644
--- a/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg
+++ b/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg
@@ -1,2 +1,3 @@
 maxtermoccurrences 100
-ilscript[0].doctype "annotationsinheritance2"
\ No newline at end of file
+fieldmatchmaxlength 1000000
+ilscript[0].doctype "annotationsinheritance2"
diff --git a/config-model/src/test/derived/annotationsreference/ilscripts.cfg b/config-model/src/test/derived/annotationsreference/ilscripts.cfg
index 2297994c727..63a1c889893 100755..100644
--- a/config-model/src/test/derived/annotationsreference/ilscripts.cfg
+++ b/config-model/src/test/derived/annotationsreference/ilscripts.cfg
@@ -1,2 +1,3 @@
 maxtermoccurrences 100
-ilscript[0].doctype "annotationsreference"
\ No newline at end of file
+fieldmatchmaxlength 1000000
+ilscript[0].doctype "annotationsreference"
diff --git a/config-model/src/test/derived/annotationssimple/ilscripts.cfg b/config-model/src/test/derived/annotationssimple/ilscripts.cfg
index a234feff99d..cb1bac309a9 100644
--- a/config-model/src/test/derived/annotationssimple/ilscripts.cfg
+++ b/config-model/src/test/derived/annotationssimple/ilscripts.cfg
@@ -1,2 +1,3 @@
 maxtermoccurrences 100
-ilscript[0].doctype "annotationssimple"
\ No newline at end of file
+fieldmatchmaxlength 1000000
+ilscript[0].doctype "annotationssimple"
diff --git a/config-model/src/test/derived/arrays/ilscripts.cfg b/config-model/src/test/derived/arrays/ilscripts.cfg
index d8fbe752676..d07f65e170f 100644
--- a/config-model/src/test/derived/arrays/ilscripts.cfg
+++ b/config-model/src/test/derived/arrays/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "arrays"
 ilscript[0].docfield[0] "tags"
 ilscript[0].docfield[1] "ratings"
@@ -9,4 +10,4 @@ ilscript[0].content[0] "clear_state | guard { input tags | for_each { tokenize n
 ilscript[0].content[1] "clear_state | guard { input ratings | summary ratings | attribute ratings; }"
 ilscript[0].content[2] "clear_state | guard { input a | tokenize normalize stem:\"SHORTEST\" | index a; }"
 ilscript[0].content[3] "clear_state | guard { input b | for_each { tokenize normalize stem:\"SHORTEST\" } | index b | attribute b; }"
-ilscript[0].content[4] "clear_state | guard { input c | for_each { tokenize normalize stem:\"SHORTEST\" } | summary c | index c; }"
\ No newline at end of file
+ilscript[0].content[4] "clear_state | guard { input c | for_each { tokenize normalize stem:\"SHORTEST\" } | summary c | index c; }"
diff --git a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg
index 201d529ce1d..cda9d7df73e 100644
--- a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg
+++ b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "prefetch"
 ilscript[0].docfield[0] "singlebyte"
 ilscript[0].docfield[1] "multibyte"
@@ -35,4 +36,4 @@ ilscript[0].content[13] "clear_state | guard { input multidouble | attribute mul
 ilscript[0].content[14] "clear_state | guard { input wsdouble | attribute wsdouble; }"
 ilscript[0].content[15] "clear_state | guard { input singlestring | attribute singlestring; }"
 ilscript[0].content[16] "clear_state | guard { input multistring | attribute multistring; }"
-ilscript[0].content[17] "clear_state | guard { input wsstring | attribute wsstring; }"
\ No newline at end of file
+ilscript[0].content[17] "clear_state | guard { input wsstring | attribute wsstring; }"
diff --git a/config-model/src/test/derived/attributes/ilscripts.cfg b/config-model/src/test/derived/attributes/ilscripts.cfg
index 450cc8e499c..4879eff9edc 100644
--- a/config-model/src/test/derived/attributes/ilscripts.cfg
+++ b/config-model/src/test/derived/attributes/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "attributes"
 ilscript[0].docfield[0] "a1"
 ilscript[0].docfield[1] "a2"
@@ -39,4 +40,4 @@ ilscript[0].content[16] "clear_state | guard { input b7 | summary b7 | attribute
 ilscript[0].content[17] "clear_state | guard { input a9 | attribute a9; }"
 ilscript[0].content[18] "clear_state | guard { input a10 | attribute a10; }"
 ilscript[0].content[19] "clear_state | guard { input a11 | attribute a11; }"
-ilscript[0].content[20] "clear_state | guard { input a12 | attribute a12; }"
\ No newline at end of file
+ilscript[0].content[20] "clear_state | guard { input a12 | attribute a12; }"
diff --git a/config-model/src/test/derived/complex/ilscripts.cfg b/config-model/src/test/derived/complex/ilscripts.cfg
index 05de728e16e..4341af4ce87 100644
--- a/config-model/src/test/derived/complex/ilscripts.cfg
+++ b/config-model/src/test/derived/complex/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "complex"
 ilscript[0].docfield[0] "title"
 ilscript[0].docfield[1] "location"
@@ -45,4 +46,4 @@ ilscript[0].content[20] "clear_state | guard { input combineda | attribute combi
 ilscript[0].content[21] "clear_state | guard { input combinedb | tokenize normalize stem:\"SHORTEST\" | index combinedb; }"
 ilscript[0].content[22] "input category | passthrough category"
 ilscript[0].content[23] "input location | passthrough location"
-ilscript[0].content[24] "input yEaR | passthrough yEaR"
\ No newline at end of file
+ilscript[0].content[24] "input yEaR | passthrough yEaR"
diff --git a/config-model/src/test/derived/emptydefault/ilscripts.cfg b/config-model/src/test/derived/emptydefault/ilscripts.cfg
index b28b5ade17f..6806ce8bf91 100644
--- a/config-model/src/test/derived/emptydefault/ilscripts.cfg
+++ b/config-model/src/test/derived/emptydefault/ilscripts.cfg
@@ -1,6 +1,7 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "emptydefault"
 ilscript[0].docfield[0] "one"
 ilscript[0].docfield[1] "two"
 ilscript[0].content[0] "clear_state | guard { input one | tokenize normalize stem:\"SHORTEST\" | index one; }"
-ilscript[0].content[1] "clear_state | guard { input two | tokenize normalize stem:\"SHORTEST\" | index two; }"
\ No newline at end of file
+ilscript[0].content[1] "clear_state | guard { input two | tokenize normalize stem:\"SHORTEST\" | index two; }"
diff --git a/config-model/src/test/derived/exactmatch/ilscripts.cfg b/config-model/src/test/derived/exactmatch/ilscripts.cfg
index c3ecab6d3a9..c5bcaf70e41 100644
--- a/config-model/src/test/derived/exactmatch/ilscripts.cfg
+++ b/config-model/src/test/derived/exactmatch/ilscripts.cfg
@@ -1,6 +1,7 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "exactmatch"
 ilscript[0].docfield[0] "tag"
 ilscript[0].docfield[1] "screweduserids"
 ilscript[0].content[0] "clear_state | guard { input tag | exact | summary tag | index tag; }"
-ilscript[0].content[1] "clear_state | guard { input screweduserids | exact | index screweduserids | summary screweduserids | attribute screweduserids; }"
\ No newline at end of file
+ilscript[0].content[1] "clear_state | guard { input screweduserids | exact | index screweduserids | summary screweduserids | attribute screweduserids; }"
diff --git a/config-model/src/test/derived/id/ilscripts.cfg b/config-model/src/test/derived/id/ilscripts.cfg
index c613431cdf9..0a617538ded 100644
--- a/config-model/src/test/derived/id/ilscripts.cfg
+++ b/config-model/src/test/derived/id/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "id"
 ilscript[0].docfield[0] "uri"
-ilscript[0].content[0] "clear_state | guard { input uri | summary uri | index uri; }"
\ No newline at end of file
+ilscript[0].content[0] "clear_state | guard { input uri | summary uri | index uri; }"
diff --git a/config-model/src/test/derived/indexswitches/ilscripts.cfg b/config-model/src/test/derived/indexswitches/ilscripts.cfg
index 706f4105224..6d1ef6b9b6d 100644
--- a/config-model/src/test/derived/indexswitches/ilscripts.cfg
+++ b/config-model/src/test/derived/indexswitches/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "indexswitches"
 ilscript[0].docfield[0] "title"
 ilscript[0].docfield[1] "descr"
@@ -6,4 +7,4 @@ ilscript[0].docfield[2] "source_src"
 ilscript[0].content[0] "clear_state | guard { input source_src | switch { case \"theweb\": input source_src | tokenize normalize | summary source | index source; case \"amg\": input source_src | tokenize normalize | summary source; default: input source_src . \" partner\" | tokenize normalize | summary source | index source; }; }"
 ilscript[0].content[1] "clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }"
 ilscript[0].content[2] "clear_state | guard { input descr | tokenize normalize stem:\"SHORTEST\" | summary descr | index descr; }"
-ilscript[0].content[3] "input source_src | passthrough source_src"
\ No newline at end of file
+ilscript[0].content[3] "input source_src | passthrough source_src"
diff --git a/config-model/src/test/derived/inheritance/ilscripts.cfg b/config-model/src/test/derived/inheritance/ilscripts.cfg
index ba5fdf58ccc..90bc86bd3f3 100644
--- a/config-model/src/test/derived/inheritance/ilscripts.cfg
+++ b/config-model/src/test/derived/inheritance/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "child"
 ilscript[0].docfield[0] "onlygrandparent"
 ilscript[0].docfield[1] "overridden"
@@ -9,4 +10,4 @@ ilscript[0].content[0] "clear_state | guard { input onlygrandparent | attribute
 ilscript[0].content[1] "clear_state | guard { input overridden | attribute overridden; }"
 ilscript[0].content[2] "clear_state | guard { input onlyfather | summary onlyfather; }"
 ilscript[0].content[3] "clear_state | guard { input onlymother | tokenize normalize stem:\"SHORTEST\" | attribute onlymother | index onlymother; }"
-ilscript[0].content[4] "clear_state | guard { input onlychild | tokenize normalize stem:\"SHORTEST\" | index onlychild; }"
\ No newline at end of file
+ilscript[0].content[4] "clear_state | guard { input onlychild | tokenize normalize stem:\"SHORTEST\" | index onlychild; }"
diff --git a/config-model/src/test/derived/music/ilscripts.cfg b/config-model/src/test/derived/music/ilscripts.cfg
index ca2c35ffcba..1df8a818484 100644
--- a/config-model/src/test/derived/music/ilscripts.cfg
+++ b/config-model/src/test/derived/music/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "music"
 ilscript[0].docfield[0] "bgndata"
 ilscript[0].docfield[1] "sales"
@@ -73,4 +74,4 @@ ilscript[0].content[33] "clear_state | guard { input cbid | summary cbid | attri
 ilscript[0].content[34] "clear_state | guard { input metalvalue | summary metalvalue; }"
 ilscript[0].content[35] "clear_state | guard { input hiphopvalue | summary hiphopvalue; }"
 ilscript[0].content[36] "clear_state | guard { input powermetalvalue | tokenize normalize stem:\"SHORTEST\" | index powermetalvalue | summary powermetalvalue; }"
-ilscript[0].content[37] "clear_state | guard { input progvalue | tokenize normalize stem:\"SHORTEST\" | index progvalue | summary progvalue; }"
\ No newline at end of file
+ilscript[0].content[37] "clear_state | guard { input progvalue | tokenize normalize stem:\"SHORTEST\" | index progvalue | summary progvalue; }"
diff --git a/config-model/src/test/derived/newrank/ilscripts.cfg b/config-model/src/test/derived/newrank/ilscripts.cfg
index 8a252be4dd0..d8f9b5101ad 100644
--- a/config-model/src/test/derived/newrank/ilscripts.cfg
+++ b/config-model/src/test/derived/newrank/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "newrank"
 ilscript[0].docfield[0] "bgndata"
 ilscript[0].docfield[1] "sales"
@@ -63,4 +64,4 @@ ilscript[0].content[27] "clear_state | guard { input bgnpto | tokenize normalize
 ilscript[0].content[28] "clear_state | guard { input year | summary year | attribute year; }"
 ilscript[0].content[29] "clear_state | guard { input did | summary did | attribute did; }"
 ilscript[0].content[30] "clear_state | guard { input scorekey | summary scorekey | attribute scorekey; }"
-ilscript[0].content[31] "clear_state | guard { input cbid | summary cbid | attribute cbid; }"
\ No newline at end of file
+ilscript[0].content[31] "clear_state | guard { input cbid | summary cbid | attribute cbid; }"
diff --git a/config-model/src/test/derived/orderilscripts/ilscripts.cfg b/config-model/src/test/derived/orderilscripts/ilscripts.cfg
index 656381d68a8..a11fbe05f5d 100755..100644
--- a/config-model/src/test/derived/orderilscripts/ilscripts.cfg
+++ b/config-model/src/test/derived/orderilscripts/ilscripts.cfg
@@ -1,5 +1,6 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "orderilscripts"
 ilscript[0].docfield[0] "foo"
 ilscript[0].content[0] "clear_state | guard { input foo | summary bar; }"
-ilscript[0].content[1] "clear_state | guard { input foo | tokenize normalize stem:\"SHORTEST\" | summary foo | index foo; }"
\ No newline at end of file
+ilscript[0].content[1] "clear_state | guard { input foo | tokenize normalize stem:\"SHORTEST\" | summary foo | index foo; }"
diff --git a/config-model/src/test/derived/position_array/ilscripts.cfg b/config-model/src/test/derived/position_array/ilscripts.cfg
index dfe3827a1d5..afdd5b548cf 100644
--- a/config-model/src/test/derived/position_array/ilscripts.cfg
+++ b/config-model/src/test/derived/position_array/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "position_array"
 ilscript[0].docfield[0] "pos"
-ilscript[0].content[0] "clear_state | guard { input pos | for_each { zcurve } | attribute pos_zcurve; }"
\ No newline at end of file
+ilscript[0].content[0] "clear_state | guard { input pos | for_each { zcurve } | attribute pos_zcurve; }"
diff --git a/config-model/src/test/derived/position_attribute/ilscripts.cfg b/config-model/src/test/derived/position_attribute/ilscripts.cfg
index ea6d6c4080b..dff9d583b3e 100644
--- a/config-model/src/test/derived/position_attribute/ilscripts.cfg
+++ b/config-model/src/test/derived/position_attribute/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "position_attribute"
 ilscript[0].docfield[0] "pos"
-ilscript[0].content[0] "clear_state | guard { input pos | zcurve | attribute pos_zcurve; }"
\ No newline at end of file
+ilscript[0].content[0] "clear_state | guard { input pos | zcurve | attribute pos_zcurve; }"
diff --git a/config-model/src/test/derived/position_extra/ilscripts.cfg b/config-model/src/test/derived/position_extra/ilscripts.cfg
index 33d61e55465..24d80d0df9d 100644
--- a/config-model/src/test/derived/position_extra/ilscripts.cfg
+++ b/config-model/src/test/derived/position_extra/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "position_extra"
 ilscript[0].docfield[0] "pos_str"
-ilscript[0].content[0] "clear_state | guard { input pos_str | to_pos | zcurve | attribute pos_ext_zcurve; }"
\ No newline at end of file
+ilscript[0].content[0] "clear_state | guard { input pos_str | to_pos | zcurve | attribute pos_ext_zcurve; }"
diff --git a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg
index 4d68178000e..eb7dadf8c1a 100644
--- a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg
+++ b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "prefixexactattribute"
 ilscript[0].docfield[0] "indexfield0"
 ilscript[0].docfield[1] "attributefield1"
@@ -9,4 +10,4 @@ ilscript[0].content[0] "clear_state | guard { input indexfield0 | tokenize norma
 ilscript[0].content[1] "clear_state | guard { input attributefield1 | attribute attributefield1; }"
 ilscript[0].content[2] "clear_state | guard { input attributefield2 | attribute attributefield2; }"
 ilscript[0].content[3] "clear_state | guard { input indexfield1 | exact | index indexfield1; }"
-ilscript[0].content[4] "clear_state | guard { input indexfield2 | exact | index indexfield2; }"
\ No newline at end of file
+ilscript[0].content[4] "clear_state | guard { input indexfield2 | exact | index indexfield2; }"
diff --git a/config-model/src/test/derived/ranktypes/ilscripts.cfg b/config-model/src/test/derived/ranktypes/ilscripts.cfg
index 3a917099bfc..ee191aafe90 100644
--- a/config-model/src/test/derived/ranktypes/ilscripts.cfg
+++ b/config-model/src/test/derived/ranktypes/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "ranktypes"
 ilscript[0].docfield[0] "title"
 ilscript[0].docfield[1] "descr"
@@ -8,4 +9,4 @@ ilscript[0].content[0] "clear_state | guard { input identity | tokenize | index
 ilscript[0].content[1] "clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }"
 ilscript[0].content[2] "clear_state | guard { input descr | tokenize normalize stem:\"SHORTEST\" | summary descr | index descr; }"
 ilscript[0].content[3] "clear_state | guard { input keywords | tokenize normalize stem:\"SHORTEST\" | index keywords; }"
-ilscript[0].content[4] "clear_state | guard { input identity | tokenize normalize stem:\"SHORTEST\" | index identity; }"
\ No newline at end of file
+ilscript[0].content[4] "clear_state | guard { input identity | tokenize normalize stem:\"SHORTEST\" | index identity; }"
diff --git a/config-model/src/test/derived/structanyorder/ilscripts.cfg b/config-model/src/test/derived/structanyorder/ilscripts.cfg
index b86571ce666..60c4ec83c01 100644
--- a/config-model/src/test/derived/structanyorder/ilscripts.cfg
+++ b/config-model/src/test/derived/structanyorder/ilscripts.cfg
@@ -1,6 +1,7 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "annotationsimplicitstruct"
 ilscript[0].docfield[0] "structfield"
 ilscript[0].docfield[1] "structarrayfield"
 ilscript[0].content[0] "input structarrayfield | passthrough structarrayfield"
-ilscript[0].content[1] "input structfield | passthrough structfield"
\ No newline at end of file
+ilscript[0].content[1] "input structfield | passthrough structfield"
diff --git a/config-model/src/test/derived/types/ilscripts.cfg b/config-model/src/test/derived/types/ilscripts.cfg
index 70b799d0df0..108fd0ca830 100644
--- a/config-model/src/test/derived/types/ilscripts.cfg
+++ b/config-model/src/test/derived/types/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "types"
 ilscript[0].docfield[0] "abyte"
 ilscript[0].docfield[1] "along"
@@ -52,4 +53,4 @@ ilscript[0].content[22] "input mystructfield | passthrough mystructfield"
 ilscript[0].content[23] "input mystructmap | passthrough mystructmap"
 ilscript[0].content[24] "input stringmapfield | passthrough stringmapfield"
 ilscript[0].content[25] "input structarrayfield | passthrough structarrayfield"
-ilscript[0].content[26] "input structfield | passthrough structfield"
\ No newline at end of file
+ilscript[0].content[26] "input structfield | passthrough structfield"
diff --git a/config-model/src/test/derived/uri_array/ilscripts.cfg b/config-model/src/test/derived/uri_array/ilscripts.cfg
index de78b44199c..721fca76b9a 100644
--- a/config-model/src/test/derived/uri_array/ilscripts.cfg
+++ b/config-model/src/test/derived/uri_array/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "uri_array"
 ilscript[0].docfield[0] "my_uri"
-ilscript[0].content[0] "clear_state | guard { input my_uri | index my_uri; }"
\ No newline at end of file
+ilscript[0].content[0] "clear_state | guard { input my_uri | index my_uri; }"
diff --git a/config-model/src/test/derived/uri_wset/ilscripts.cfg b/config-model/src/test/derived/uri_wset/ilscripts.cfg
index 81066cd130d..5d32c7b2e63 100644
--- a/config-model/src/test/derived/uri_wset/ilscripts.cfg
+++ b/config-model/src/test/derived/uri_wset/ilscripts.cfg
@@ -1,4 +1,5 @@
 maxtermoccurrences 100
+fieldmatchmaxlength 1000000
 ilscript[0].doctype "uri_wset"
 ilscript[0].docfield[0] "my_uri"
-ilscript[0].content[0] "clear_state | guard { input my_uri | index my_uri; }"
\ No newline at end of file
+ilscript[0].content[0] "clear_state | guard { input my_uri | index my_uri; }"
diff --git a/configdefinitions/src/vespa/ilscripts.def b/configdefinitions/src/vespa/ilscripts.def
index 316e046800b..2fe34f678cc 100644
--- a/configdefinitions/src/vespa/ilscripts.def
+++ b/configdefinitions/src/vespa/ilscripts.def
@@ -4,6 +4,7 @@ namespace=vespa.configdefinition
 
 ## The maximum number of occurrences of a given term to index per field
 maxtermoccurrences int default=100
+fieldmatchmaxlength int default=1000000
 
 ilscript[].doctype    string
 ilscript[].docfield[] string
diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java
index 015a5ceb79d..b1ec78fa0c4 100644
--- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java
+++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java
@@ -76,6 +76,7 @@ public class ScriptManager {
         Map<String, Map<String, DocumentScript>> documentFieldScripts = new HashMap<>(config.ilscript().size());
         ScriptParserContext parserContext = new ScriptParserContext(linguistics);
         parserContext.getAnnotatorConfig().setMaxTermOccurrences(config.maxtermoccurrences());
+        parserContext.getAnnotatorConfig().setMaxTokenLength(config.fieldmatchmaxlength());
 
         for (IlscriptsConfig.Ilscript ilscript : config.ilscript()) {
             InputExpression.FieldPathOptimizer fieldPathOptimizer = new InputExpression.FieldPathOptimizer(docTypeMgr.getDocumentType(ilscript.doctype()));
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java
index 990a8a513f2..6d160c489df 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java
@@ -14,14 +14,15 @@ public class AnnotatorConfig implements Cloneable {
     private StemMode stemMode;
     private boolean removeAccents;
     private int maxTermOccurences;
-    private int maxTokenizeLength = MAX_TOKENIZE_LENGTH;
+    private int maxTokenizeLength;
 
     public static final int DEFAULT_MAX_TERM_OCCURRENCES;
-    private static final int MAX_TOKENIZE_LENGTH = 1000000;
+    private static final int DEFAULT_MAX_TOKENIZE_LENGTH;
 
     static {
         IlscriptsConfig defaults = new IlscriptsConfig(new IlscriptsConfig.Builder());
         DEFAULT_MAX_TERM_OCCURRENCES = defaults.maxtermoccurrences();
+        DEFAULT_MAX_TOKENIZE_LENGTH = defaults.fieldmatchmaxlength();
     }
 
     public AnnotatorConfig() {
@@ -29,6 +30,7 @@ public class AnnotatorConfig implements Cloneable {
         stemMode = StemMode.NONE;
         removeAccents = false;
         maxTermOccurences = DEFAULT_MAX_TERM_OCCURRENCES;
+        maxTokenizeLength = DEFAULT_MAX_TOKENIZE_LENGTH;
     }
 
     public AnnotatorConfig(AnnotatorConfig rhs) {
@@ -90,7 +92,7 @@ public class AnnotatorConfig implements Cloneable {
     }
 
     public boolean hasNonDefaultMaxTokenLength() {
-        return maxTokenizeLength != MAX_TOKENIZE_LENGTH;
+        return maxTokenizeLength != DEFAULT_MAX_TOKENIZE_LENGTH;
     }
 
     @Override