Revert "Drop tokenize expressions from ilscript for streaming mode."

2 files changed, 7 insertions, 28 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java
index 575b7264628..7f874d04f13 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java
@@ -80,7 +80,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer {
                 summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags());
                 juniperrc = new Juniperrc(schema);
                 rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState);
-                indexingScript = new IndexingScript(schema, isStreaming);
+                indexingScript = new IndexingScript(schema);
                 indexInfo = new IndexInfo(schema, isStreaming);
                 schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries);
                 indexSchema = new IndexSchema(schema);
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java
index 6f0420f5203..73f3507ab00 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java
@@ -6,7 +6,6 @@ import com.yahoo.schema.document.GeoPos;
 import com.yahoo.schema.document.ImmutableSDField;
 import com.yahoo.vespa.configdefinition.IlscriptsConfig;
 import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder;
-import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
 import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
 import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.Expression;
@@ -17,7 +16,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.PassthroughExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
-import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression;
 
 import java.util.ArrayList;
@@ -38,10 +36,8 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
     private final List<String> docFields = new ArrayList<>();
     private final List<Expression> expressions = new ArrayList<>();
     private List<ImmutableSDField> fieldsSettingLanguage;
-    private final boolean isStreaming;
 
-    public IndexingScript(Schema schema, boolean isStreaming) {
-        this.isStreaming = isStreaming;
+    public IndexingScript(Schema schema) {
         derive(schema);
     }
 
@@ -100,38 +96,21 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
         configBuilder.ilscript(ilscriptBuilder);
     }
 
-    private static class DropTokenize extends ExpressionConverter {
-        @Override
-        protected boolean shouldConvert(Expression exp) {
-            return exp instanceof TokenizeExpression;
-        }
-
-        @Override
-        protected Expression doConvert(Expression exp) {
-            return null;
-        }
-    }
-
     private void addContentInOrder(IlscriptsConfig.Ilscript.Builder ilscriptBuilder) {
         ArrayList<Expression> later = new ArrayList<>();
         Set<String> touchedFields = new HashSet<>();
         for (Expression expression : expressions) {
-            if (isStreaming) {
-                expression = expression.convertChildren(new DropTokenize());
-            }
-            if (modifiesSelf(expression) && ! setsLanguage(expression)) {
+            if (modifiesSelf(expression) && ! setsLanguage(expression))
                 later.add(expression);
-            } else {
+            else
                 ilscriptBuilder.content(expression.toString());
-            }
 
             FieldScanVisitor fieldFetcher = new FieldScanVisitor();
             fieldFetcher.visit(expression);
             touchedFields.addAll(fieldFetcher.touchedFields());
         }
-        for (Expression exp : later) {
+        for (Expression exp : later)
             ilscriptBuilder.content(exp.toString());
-        }
         generateSyntheticStatementsForUntouchedFields(ilscriptBuilder, touchedFields);
     }
 
@@ -192,8 +171,8 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
     }
 
     private static class FieldScanVisitor extends ExpressionVisitor {
-        List<String> touchedFields = new ArrayList<>();
-        List<String> candidates = new ArrayList<>();
+        List<String> touchedFields = new ArrayList<String>();
+        List<String> candidates = new ArrayList<String>();
 
         @Override
         protected void doVisit(Expression exp) {