Merge pull request #29195 from vespa-engine/toregge/dynamic-summary-fields-dont-require-tokenizer

Dynamic summary fields don't require tokenizer.

7 files changed, 16 insertions, 32 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java
index 7dd968c5454..8ae3ec7a3fa 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java
@@ -47,13 +47,7 @@ public class TextMatch extends Processor {
             }
             if (fieldType != DataType.STRING) continue;
 
-            Set<String> dynamicSummary = new TreeSet<>();
-            Set<String> staticSummary = new TreeSet<>();
-            new IndexingOutputs(schema, deployLogger, rankProfileRegistry, queryProfiles).findSummaryTo(schema,
-                                                                                                        field,
-                                                                                                        dynamicSummary,
-                                                                                                        staticSummary);
-            MyVisitor visitor = new MyVisitor(dynamicSummary);
+            MyVisitor visitor = new MyVisitor();
             visitor.visit(script);
             if ( ! visitor.requiresTokenize) continue;
 
@@ -78,23 +72,15 @@ public class TextMatch extends Processor {
 
     private static class MyVisitor extends ExpressionVisitor {
 
-        final Set<String> dynamicSummaryFields;
         boolean requiresTokenize = false;
 
-        MyVisitor(Set<String> dynamicSummaryFields) {
-            this.dynamicSummaryFields = dynamicSummaryFields;
-        }
+        MyVisitor() { }
 
         @Override
         protected void doVisit(Expression exp) {
             if (exp instanceof IndexExpression) {
                 requiresTokenize = true;
             }
-            if (exp instanceof SummaryExpression &&
-                dynamicSummaryFields.contains(((SummaryExpression)exp).getFieldName()))
-            {
-                requiresTokenize = true;
-            }
         }
 
     }
diff --git a/config-model/src/test/derived/complex/ilscripts.cfg b/config-model/src/test/derived/complex/ilscripts.cfg
index f7f6c9dd720..6074333bd24 100644
--- a/config-model/src/test/derived/complex/ilscripts.cfg
+++ b/config-model/src/test/derived/complex/ilscripts.cfg
@@ -27,7 +27,7 @@ ilscript[].content[] "clear_state | guard { input location | tokenize normalize
 ilscript[].content[] "clear_state | guard { input yEaR | to_array | attribute year_arr; }"
 ilscript[].content[] "clear_state | guard { input yEaR - 1900 | attribute year_sub; }"
 ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | index title | summary title; }"
-ilscript[].content[] "clear_state | guard { input dyntitle | tokenize normalize stem:\"BEST\" | summary dyntitle; }"
+ilscript[].content[] "clear_state | guard { input dyntitle | summary dyntitle; }"
 ilscript[].content[] "clear_state | guard { input special1 | tokenize normalize | index special1; }"
 ilscript[].content[] "clear_state | guard { input special2 | tokenize normalize | index special2; }"
 ilscript[].content[] "clear_state | guard { input special3 | tokenize normalize | index special3; }"
diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg
index 6429932ab0e..1993bbd14ac 100644
--- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg
+++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg
@@ -14,13 +14,13 @@ ilscript[].docfield[] "h"
 ilscript[].docfield[] "loc"
 ilscript[].docfield[] "mytags"
 ilscript[].content[] "clear_state | guard { input loc | to_pos | zcurve | attribute loc_pos_zcurve; }"
-ilscript[].content[] "clear_state | guard { input a | tokenize normalize stem:\"BEST\" | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a; }"
-ilscript[].content[] "clear_state | guard { input adynamic | tokenize normalize stem:\"BEST\" | summary adynamic | attribute adynamic; }"
-ilscript[].content[] "clear_state | guard { input abolded | tokenize normalize stem:\"BEST\" | summary abolded | attribute abolded; }"
+ilscript[].content[] "clear_state | guard { input a | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a; }"
+ilscript[].content[] "clear_state | guard { input adynamic | summary adynamic | attribute adynamic; }"
+ilscript[].content[] "clear_state | guard { input abolded | summary abolded | attribute abolded; }"
 ilscript[].content[] "clear_state | guard { input b | summary anotherb | summary b; }"
 ilscript[].content[] "clear_state | guard { input c | summary c | attribute c; }"
-ilscript[].content[] "clear_state | guard { input d | tokenize normalize stem:\"BEST\" | summary d; }"
-ilscript[].content[] "clear_state | guard { input e | tokenize normalize stem:\"BEST\" | summary dynamice | summary e; }"
+ilscript[].content[] "clear_state | guard { input d | summary d; }"
+ilscript[].content[] "clear_state | guard { input e | summary dynamice | summary e; }"
 ilscript[].content[] "clear_state | guard { input f | summary f; }"
 ilscript[].content[] "clear_state | guard { input g | summary g; }"
 ilscript[].content[] "clear_state | guard { input h | summary h; }"
diff --git a/config-model/src/test/derived/music/ilscripts.cfg b/config-model/src/test/derived/music/ilscripts.cfg
index 7a02d836db5..ba292c4013a 100644
--- a/config-model/src/test/derived/music/ilscripts.cfg
+++ b/config-model/src/test/derived/music/ilscripts.cfg
@@ -39,7 +39,7 @@ ilscript[].docfield[] "powermetalvalue"
 ilscript[].docfield[] "progvalue"
 ilscript[].content[] "clear_state | guard { input hiphopvalue | split \";\" | attribute hiphopvalue_arr; }"
 ilscript[].content[] "clear_state | guard { input metalvalue | split \";\" | attribute metalvalue_arr; }"
-ilscript[].content[] "clear_state | guard { input bgndata | tokenize normalize stem:\"BEST\" | summary bgndata; }"
+ilscript[].content[] "clear_state | guard { input bgndata | summary bgndata; }"
 ilscript[].content[] "clear_state | guard { input sales | summary sales | attribute sales; }"
 ilscript[].content[] "clear_state | guard { input pto | summary pto | attribute pto; }"
 ilscript[].content[] "clear_state | guard { input keys | tokenize normalize stem:\"BEST\" | index keys; }"
@@ -66,7 +66,7 @@ ilscript[].content[] "clear_state | guard { input artist | tokenize normalize st
 ilscript[].content[] "clear_state | guard { input artistspid | summary artistspid; }"
 ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }"
 ilscript[].content[] "clear_state | guard { input newestedition | summary newestedition | attribute newestedition; }"
-ilscript[].content[] "clear_state | guard { input bgnpto | tokenize normalize stem:\"BEST\" | summary bgnpto; }"
+ilscript[].content[] "clear_state | guard { input bgnpto | summary bgnpto; }"
 ilscript[].content[] "clear_state | guard { input year | summary year | attribute year; }"
 ilscript[].content[] "clear_state | guard { input did | summary did | attribute did; }"
 ilscript[].content[] "clear_state | guard { input scorekey | summary scorekey; }"
diff --git a/config-model/src/test/derived/newrank/ilscripts.cfg b/config-model/src/test/derived/newrank/ilscripts.cfg
index 6986f12f62a..ec46d9acc68 100644
--- a/config-model/src/test/derived/newrank/ilscripts.cfg
+++ b/config-model/src/test/derived/newrank/ilscripts.cfg
@@ -33,7 +33,7 @@ ilscript[].docfield[] "year"
 ilscript[].docfield[] "did"
 ilscript[].docfield[] "scorekey"
 ilscript[].docfield[] "cbid"
-ilscript[].content[] "clear_state | guard { input bgndata | tokenize normalize stem:\"BEST\" | summary bgndata; }"
+ilscript[].content[] "clear_state | guard { input bgndata | summary bgndata; }"
 ilscript[].content[] "clear_state | guard { input sales | summary sales | attribute sales; }"
 ilscript[].content[] "clear_state | guard { input pto | summary pto | attribute pto; }"
 ilscript[].content[] "clear_state | guard { input keys | tokenize normalize stem:\"BEST\" | index keys; }"
@@ -60,7 +60,7 @@ ilscript[].content[] "clear_state | guard { input artist | tokenize normalize st
 ilscript[].content[] "clear_state | guard { input artistspid | summary artistspid; }"
 ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }"
 ilscript[].content[] "clear_state | guard { input newestedition | summary newestedition | attribute newestedition; }"
-ilscript[].content[] "clear_state | guard { input bgnpto | tokenize normalize stem:\"BEST\" | summary bgnpto; }"
+ilscript[].content[] "clear_state | guard { input bgnpto | summary bgnpto; }"
 ilscript[].content[] "clear_state | guard { input year | summary year | attribute year; }"
 ilscript[].content[] "clear_state | guard { input did | summary did | attribute did; }"
 ilscript[].content[] "clear_state | guard { input scorekey | summary scorekey | attribute scorekey; }"
diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java
index 62f36a37d87..b62fc61f5b7 100644
--- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java
@@ -48,7 +48,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase {
     void testDynamicSummaryRewriting() {
         SDField field = createField("test", DataType.STRING, "{ summary }");
         field.addSummaryField(createDynamicSummaryField(field, "dyn"));
-        assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary dyn; }", field);
+        assertIndexingScript("{ input test | summary dyn; }", field);
     }
 
     @Test
@@ -113,7 +113,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase {
                         "clear_state | guard { input chatter | tokenize normalize stem:\"BEST\" | index chatter; }",
                         "clear_state | guard { input description | tokenize normalize stem:\"BEST\" | summary description | summary dyndesc | index description; }",
                         "clear_state | guard { input exactemento_src | lowercase | tokenize normalize stem:\"BEST\" | index exactemento | summary exactemento; }",
-                        "clear_state | guard { input longdesc | tokenize normalize stem:\"BEST\" | summary dyndesc2 | summary dynlong | summary longdesc | summary longstat; }",
+                        "clear_state | guard { input longdesc | summary dyndesc2 | summary dynlong | summary longdesc | summary longstat; }",
                         "clear_state | guard { input measurement | attribute measurement | summary measurement; }",
                         "clear_state | guard { input measurement | to_array | attribute measurement_arr; }",
                         "clear_state | guard { input popularity | attribute popularity; }",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java
index 0c2a26d9c1d..6b58cac3f6c 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java
@@ -116,12 +116,10 @@ public class IndexingScriptChangeValidatorTest {
     }
 
     @Test
-    void requireThatSettingDynamicSummaryRequireReindexing() throws Exception {
+    void requireThatSettingDynamicSummaryIsOk() throws Exception {
         new Fixture(FIELD + " { indexing: summary }",
                 FIELD + " { indexing: summary \n summary: dynamic }").
-                assertValidation(expectedReindexingAction("summary field 'f1' transform: 'none' -> 'dynamicteaser'",
-                "{ input f1 | summary f1; }",
-                "{ input f1 | tokenize normalize stem:\"BEST\" | summary f1; }"));
+                assertValidation();
     }
 
     @Test