Dynamic summary fields don't require tokenizer.

Tokenization is enabled by the presence of an index expression in the
indexing script for the field, and the annotated strings containing the
result of tokenization are written to the document field.

7 files changed, 16 insertions, 32 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java
index 7dd968c5454..8ae3ec7a3fa 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java
@@ -47,13 +47,7 @@ public class TextMatch extends Processor {
             }
             if (fieldType != DataType.STRING) continue;
 
-            Set<String> dynamicSummary = new TreeSet<>();
-            Set<String> staticSummary = new TreeSet<>();
-            new IndexingOutputs(schema, deployLogger, rankProfileRegistry, queryProfiles).findSummaryTo(schema,
-                                                                                                        field,
-                                                                                                        dynamicSummary,
-                                                                                                        staticSummary);
-            MyVisitor visitor = new MyVisitor(dynamicSummary);
+            MyVisitor visitor = new MyVisitor();
             visitor.visit(script);
             if ( ! visitor.requiresTokenize) continue;
 
@@ -78,23 +72,15 @@ public class TextMatch extends Processor {
 
     private static class MyVisitor extends ExpressionVisitor {
 
-        final Set<String> dynamicSummaryFields;
         boolean requiresTokenize = false;
 
-        MyVisitor(Set<String> dynamicSummaryFields) {
-            this.dynamicSummaryFields = dynamicSummaryFields;
-        }
+        MyVisitor() { }
 
         @Override
         protected void doVisit(Expression exp) {
             if (exp instanceof IndexExpression) {
                 requiresTokenize = true;
             }
-            if (exp instanceof SummaryExpression &&
-                dynamicSummaryFields.contains(((SummaryExpression)exp).getFieldName()))
-            {
-                requiresTokenize = true;
-            }
         }
 
     }
diff --git a/config-model/src/test/derived/complex/ilscripts.cfg b/config-model/src/test/derived/complex/ilscripts.cfg
index f7f6c9dd720..6074333bd24 100644
--- a/config-model/src/test/derived/complex/ilscripts.cfg
+++ b/config-model/src/test/derived/complex/ilscripts.cfg
@@ -27,7 +27,7 @@ ilscript[].content[] "clear_state | guard { input location | tokenize normalize
 ilscript[].content[] "clear_state | guard { input yEaR | to_array | attribute year_arr; }"
 ilscript[].content[] "clear_state | guard { input yEaR - 1900 | attribute year_sub; }"
 ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | index title | summary title; }"
-ilscript[].content[] "clear_state | guard { input dyntitle | tokenize normalize stem:\"BEST\" | summary dyntitle; }"
+ilscript[].content[] "clear_state | guard { input dyntitle | summary dyntitle; }"
 ilscript[].content[] "clear_state | guard { input special1 | tokenize normalize | index special1; }"
 ilscript[].content[] "clear_state | guard { input special2 | tokenize normalize | index special2; }"
 ilscript[].content[] "clear_state | guard { input special3 | tokenize normalize | index special3; }"
diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg
index 6429932ab0e..1993bbd14ac 100644
--- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg
+++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg
@@ -14,13 +14,13 @@ ilscript[].docfield[] "h"
 ilscript[].docfield[] "loc"
 ilscript[].docfield[] "mytags"
 ilscript[].content[] "clear_state | guard { input loc | to_pos | zcurve | attribute loc_pos_zcurve; }"
-ilscript[].content[] "clear_state | guard { input a | tokenize normalize stem:\"BEST\" | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a; }"
-ilscript[].content[] "clear_state | guard { input adynamic | tokenize normalize stem:\"BEST\" | summary adynamic | attribute adynamic; }"
-ilscript[].content[] "clear_state | guard { input abolded | tokenize normalize stem:\"BEST\" | summary abolded | attribute abolded; }"
+ilscript[].content[] "clear_state | guard { input a | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a; }"
+ilscript[].content[] "clear_state | guard { input adynamic | summary adynamic | attribute adynamic; }"
+ilscript[].content[] "clear_state | guard { input abolded | summary abolded | attribute abolded; }"
 ilscript[].content[] "clear_state | guard { input b | summary anotherb | summary b; }"
 ilscript[].content[] "clear_state | guard { input c | summary c | attribute c; }"
-ilscript[].content[] "clear_state | guard { input d | tokenize normalize stem:\"BEST\" | summary d; }"
-ilscript[].content[] "clear_state | guard { input e | tokenize normalize stem:\"BEST\" | summary dynamice | summary e; }"
+ilscript[].content[] "clear_state | guard { input d | summary d; }"
+ilscript[].content[] "clear_state | guard { input e | summary dynamice | summary e; }"
 ilscript[].content[] "clear_state | guard { input f | summary f; }"
 ilscript[].content[] "clear_state | guard { input g | summary g; }"
 ilscript[].content[] "clear_state | guard { input h | summary h; }"
diff --git a/config-model/src/test/derived/music/ilscripts.cfg b/config-model/src/test/derived/music/ilscripts.cfg
index 7a02d836db5..ba292c4013a 100644
--- a/config-model/src/test/derived/music/ilscripts.cfg
+++ b/config-model/src/test/derived/music/ilscripts.cfg
@@ -39,7 +39,7 @@ ilscript[].docfield[] "powermetalvalue"
 ilscript[].docfield[] "progvalue"
 ilscript[].content[] "clear_state | guard { input hiphopvalue | split \";\" | attribute hiphopvalue_arr; }"
 ilscript[].content[] "clear_state | guard { input metalvalue | split \";\" | attribute metalvalue_arr; }"
-ilscript[].content[] "clear_state | guard { input bgndata | tokenize normalize stem:\"BEST\" | summary bgndata; }"
+ilscript[].content[] "clear_state | guard { input bgndata | summary bgndata; }"
 ilscript[].content[] "clear_state | guard { input sales | summary sales | attribute sales; }"
 ilscript[].content[] "clear_state | guard { input pto | summary pto | attribute pto; }"
 ilscript[].content[] "clear_state | guard { input keys | tokenize normalize stem:\"BEST\" | index keys; }"
@@ -66,7 +66,7 @@ ilscript[].content[] "clear_state | guard { input artist | tokenize normalize st
 ilscript[].content[] "clear_state | guard { input artistspid | summary artistspid; }"
 ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }"
 ilscript[].content[] "clear_state | guard { input newestedition | summary newestedition | attribute newestedition; }"
-ilscript[].content[] "clear_state | guard { input bgnpto | tokenize normalize stem:\"BEST\" | summary bgnpto; }"
+ilscript[].content[] "clear_state | guard { input bgnpto | summary bgnpto; }"
 ilscript[].content[] "clear_state | guard { input year | summary year | attribute year; }"
 ilscript[].content[] "clear_state | guard { input did | summary did | attribute did; }"
 ilscript[].content[] "clear_state | guard { input scorekey | summary scorekey; }"
diff --git a/config-model/src/test/derived/newrank/ilscripts.cfg b/config-model/src/test/derived/newrank/ilscripts.cfg
index 6986f12f62a..ec46d9acc68 100644
--- a/config-model/src/test/derived/newrank/ilscripts.cfg
+++ b/config-model/src/test/derived/newrank/ilscripts.cfg
@@ -33,7 +33,7 @@ ilscript[].docfield[] "year"
 ilscript[].docfield[] "did"
 ilscript[].docfield[] "scorekey"
 ilscript[].docfield[] "cbid"
-ilscript[].content[] "clear_state | guard { input bgndata | tokenize normalize stem:\"BEST\" | summary bgndata; }"
+ilscript[].content[] "clear_state | guard { input bgndata | summary bgndata; }"
 ilscript[].content[] "clear_state | guard { input sales | summary sales | attribute sales; }"
 ilscript[].content[] "clear_state | guard { input pto | summary pto | attribute pto; }"
 ilscript[].content[] "clear_state | guard { input keys | tokenize normalize stem:\"BEST\" | index keys; }"
@@ -60,7 +60,7 @@ ilscript[].content[] "clear_state | guard { input artist | tokenize normalize st
 ilscript[].content[] "clear_state | guard { input artistspid | summary artistspid; }"
 ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }"
 ilscript[].content[] "clear_state | guard { input newestedition | summary newestedition | attribute newestedition; }"
-ilscript[].content[] "clear_state | guard { input bgnpto | tokenize normalize stem:\"BEST\" | summary bgnpto; }"
+ilscript[].content[] "clear_state | guard { input bgnpto | summary bgnpto; }"
 ilscript[].content[] "clear_state | guard { input year | summary year | attribute year; }"
 ilscript[].content[] "clear_state | guard { input did | summary did | attribute did; }"
 ilscript[].content[] "clear_state | guard { input scorekey | summary scorekey | attribute scorekey; }"
diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java
index 62f36a37d87..b62fc61f5b7 100644
--- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java
@@ -48,7 +48,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase {
     void testDynamicSummaryRewriting() {
         SDField field = createField("test", DataType.STRING, "{ summary }");
         field.addSummaryField(createDynamicSummaryField(field, "dyn"));
-        assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary dyn; }", field);
+        assertIndexingScript("{ input test | summary dyn; }", field);
     }
 
     @Test
@@ -113,7 +113,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase {
                         "clear_state | guard { input chatter | tokenize normalize stem:\"BEST\" | index chatter; }",
                         "clear_state | guard { input description | tokenize normalize stem:\"BEST\" | summary description | summary dyndesc | index description; }",
                         "clear_state | guard { input exactemento_src | lowercase | tokenize normalize stem:\"BEST\" | index exactemento | summary exactemento; }",
-                        "clear_state | guard { input longdesc | tokenize normalize stem:\"BEST\" | summary dyndesc2 | summary dynlong | summary longdesc | summary longstat; }",
+                        "clear_state | guard { input longdesc | summary dyndesc2 | summary dynlong | summary longdesc | summary longstat; }",
                         "clear_state | guard { input measurement | attribute measurement | summary measurement; }",
                         "clear_state | guard { input measurement | to_array | attribute measurement_arr; }",
                         "clear_state | guard { input popularity | attribute popularity; }",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java
index 0c2a26d9c1d..6b58cac3f6c 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java
@@ -116,12 +116,10 @@ public class IndexingScriptChangeValidatorTest {
     }
 
     @Test
-    void requireThatSettingDynamicSummaryRequireReindexing() throws Exception {
+    void requireThatSettingDynamicSummaryIsOk() throws Exception {
         new Fixture(FIELD + " { indexing: summary }",
                 FIELD + " { indexing: summary \n summary: dynamic }").
-                assertValidation(expectedReindexingAction("summary field 'f1' transform: 'none' -> 'dynamicteaser'",
-                "{ input f1 | summary f1; }",
-                "{ input f1 | tokenize normalize stem:\"BEST\" | summary f1; }"));
+                assertValidation();
     }
 
     @Test