Drop tokenize expressions from ilscript for streaming mode.

2 files changed, 41 insertions, 1 deletions

diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IndexingScriptTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IndexingScriptTestCase.java
new file mode 100644
index 00000000000..6e86a567e68
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/schema/derived/IndexingScriptTestCase.java
@@ -0,0 +1,40 @@
+package com.yahoo.schema.derived;
+
+import com.yahoo.document.DataType;
+import com.yahoo.schema.Schema;
+import com.yahoo.schema.document.MatchType;
+import com.yahoo.schema.document.Matching;
+import com.yahoo.schema.document.SDField;
+import com.yahoo.schema.document.TemporarySDField;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class IndexingScriptTestCase {
+
+    private static IlscriptsConfig ilscriptsConfig(Schema schema, boolean isStreaming) {
+        IndexingScript script = new IndexingScript(schema, isStreaming);
+        IlscriptsConfig.Builder cfgBuilder = new IlscriptsConfig.Builder();
+        script.getConfig(cfgBuilder);
+        return cfgBuilder.build();
+    }
+
+    private void verifyIndexingScript(boolean isStreaming) {
+        Schema schema = VsmFieldsTestCase.createSchema();
+        SDField field = new TemporarySDField(schema.getDocument(), "f", DataType.STRING);
+        field.parseIndexingScript("{ tokenize | index }");
+        field.setMatching(new Matching(MatchType.TEXT));
+        schema.getDocument().addField(field);
+        IlscriptsConfig cfg = ilscriptsConfig(schema, isStreaming);
+        assertEquals(1, cfg.ilscript().size());
+        assertEquals(2, cfg.ilscript(0).content().size());
+        String indexing = isStreaming ? "index" : "tokenize | index";
+        assertEquals("clear_state | guard { " + indexing + "; }", cfg.ilscript(0).content(0));
+    }
+    @Test
+    void testThatTokenizeIsIgnoredFromStreaming() {
+        verifyIndexingScript(false);
+        verifyIndexingScript(true);
+    }
+}
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java
index e469b80ab40..042feeb5f46 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java
@@ -23,7 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
  */
 public class VsmFieldsTestCase {
 
-    private static Schema createSchema() {
+     static Schema createSchema() {
         Schema schema = new Schema("test", MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties());
         var sdoc = new SDDocumentType("test");
         schema.addDocument(sdoc);