Also test and handle array<string>

6 files changed, 68 insertions, 62 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java
index 6f0420f5203..375a0fa3700 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
 import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
 import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.GuardExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.InputExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression;
@@ -103,7 +104,9 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
     private static class DropTokenize extends ExpressionConverter {
         @Override
         protected boolean shouldConvert(Expression exp) {
-            return exp instanceof TokenizeExpression;
+            // Handle both string and array<string>
+            return (exp instanceof TokenizeExpression) ||
+                    ((exp instanceof ForEachExpression foreach) && (foreach.getInnerExpression() instanceof TokenizeExpression));
         }
 
         @Override
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java
index 34dcc9139b3..8ccc8870419 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java
@@ -6,7 +6,11 @@ import com.yahoo.document.Field;
 import com.yahoo.schema.Schema;
 import com.yahoo.schema.document.Attribute;
 import com.yahoo.vespa.indexinglanguage.ValueTransformProvider;
-import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression;
 
 /**
  * @author Simon Thoresen Hult
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java
index 09450fa8023..1643cf4d541 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java
@@ -45,7 +45,7 @@ public class IndexInfoTestCase {
         return builder.build();
     }
 
-    private static VespaModel createModel(String schemaName, String sdContent) {
+    static VespaModel createModel(String schemaName, String sdContent) {
         var builder = new DeployState.Builder();
         return new ApplicationPackageBuilder()
                 .addCluster(new ContentClusterBuilder().name("content").docTypes(List.of(DocType.index(schemaName))))
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IndexingScriptTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IndexingScriptTestCase.java
index 81536bf99fb..6bfb67b3011 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/IndexingScriptTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/IndexingScriptTestCase.java
@@ -1,19 +1,17 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.schema.derived;
 
-import com.yahoo.document.DataType;
 import com.yahoo.schema.Schema;
-import com.yahoo.schema.document.MatchType;
-import com.yahoo.schema.document.Matching;
-import com.yahoo.schema.document.SDField;
-import com.yahoo.schema.document.TemporarySDField;
 import com.yahoo.vespa.configdefinition.IlscriptsConfig;
+import com.yahoo.vespa.model.VespaModel;
 import org.junit.jupiter.api.Test;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class IndexingScriptTestCase {
 
+    private static final String TEST = "test";
+
     private static IlscriptsConfig ilscriptsConfig(Schema schema, boolean isStreaming) {
         IndexingScript script = new IndexingScript(schema, isStreaming);
         IlscriptsConfig.Builder cfgBuilder = new IlscriptsConfig.Builder();
@@ -22,17 +20,21 @@ public class IndexingScriptTestCase {
     }
 
     private void verifyIndexingScript(boolean isStreaming) {
-        Schema schema = VsmFieldsTestCase.createSchema();
-        SDField field = new TemporarySDField(schema.getDocument(), "f", DataType.STRING);
-        field.parseIndexingScript("{ tokenize | index }");
-        field.setMatching(new Matching(MatchType.TEXT));
-        schema.getDocument().addField(field);
+        VespaModel model = IndexInfoTestCase.createModel(TEST,
+                """
+                        field f type string { indexing: index }
+                        field fa type array<string> { indexing: index }
+                        """);
+        Schema schema = model.getSearchClusters().get(0).schemas().get(TEST).fullSchema();
         IlscriptsConfig cfg = ilscriptsConfig(schema, isStreaming);
         assertEquals(1, cfg.ilscript().size());
         assertEquals(2, cfg.ilscript(0).content().size());
-        String indexing = isStreaming ? "index" : "tokenize | index";
-        assertEquals("clear_state | guard { " + indexing + "; }", cfg.ilscript(0).content(0));
+        String exp_f = isStreaming ? "" : "tokenize normalize stem:\"BEST\" | ";
+        String exp_fa = isStreaming ? "" : "for_each { tokenize normalize stem:\"BEST\" } | ";
+        assertEquals("clear_state | guard { input f | " + exp_f + "index f; }", cfg.ilscript(0).content(0));
+        assertEquals("clear_state | guard { input fa | " + exp_fa + "index fa; }", cfg.ilscript(0).content(1));
     }
+
     @Test
     void testThatTokenizeIsIgnoredFromStreaming() {
         verifyIndexingScript(false);
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java
index 042feeb5f46..423bc0b1798 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java
@@ -28,54 +28,52 @@ public class VsmFieldsTestCase {
         var sdoc = new SDDocumentType("test");
         schema.addDocument(sdoc);
         return schema;
-    }
-
-    private static VsmfieldsConfig vsmfieldsConfig(Schema schema) {
-        VsmFields vsmFields = new VsmFields(schema);
-        VsmfieldsConfig.Builder cfgBuilder = new VsmfieldsConfig.Builder();
-        vsmFields.getConfig(cfgBuilder);
-        return cfgBuilder.build();
-    }
+     }
+     private static VsmfieldsConfig vsmfieldsConfig(Schema schema) {
+         VsmFields vsmFields = new VsmFields(schema);
+         VsmfieldsConfig.Builder cfgBuilder = new VsmfieldsConfig.Builder();
+         vsmFields.getConfig(cfgBuilder);return cfgBuilder.build();
+     }
 
     @Test
-    void reference_type_field_is_unsearchable() {
-        Schema schema = createSchema();
-        SDField field = new TemporarySDField(schema.getDocument(), "ref_field", NewDocumentReferenceDataType.forDocumentName("parent_type"));
-        field.parseIndexingScript("{ summary }");
-        schema.getDocument().addField(field);
-        VsmfieldsConfig cfg = vsmfieldsConfig(schema);
+     void reference_type_field_is_unsearchable() {
+         Schema schema = createSchema();
+         SDField field = new TemporarySDField(schema.getDocument(), "ref_field", NewDocumentReferenceDataType.forDocumentName("parent_type"));
+         field.parseIndexingScript("{ summary }");
+         schema.getDocument().addField(field);
+         VsmfieldsConfig cfg = vsmfieldsConfig(schema);
 
-        assertEquals(1, cfg.fieldspec().size());
-        VsmfieldsConfig.Fieldspec fieldSpec = cfg.fieldspec().get(0);
-        assertEquals("ref_field", fieldSpec.name());
-        assertEquals(VsmfieldsConfig.Fieldspec.Searchmethod.NONE, fieldSpec.searchmethod());
-    }
+         assertEquals(1, cfg.fieldspec().size());
+         VsmfieldsConfig.Fieldspec fieldSpec = cfg.fieldspec().get(0);
+         assertEquals("ref_field", fieldSpec.name());
+         assertEquals(VsmfieldsConfig.Fieldspec.Searchmethod.NONE, fieldSpec.searchmethod());
+     }
 
-    private void testIndexMatching(Matching matching, VsmfieldsConfig.Fieldspec.Normalize.Enum normalize, String arg1) {
-        Schema schema = createSchema();
-        SDField field = new TemporarySDField(schema.getDocument(), "f", DataType.STRING);
-        field.parseIndexingScript("{ index }");
-        field.setMatching(matching);
-        schema.getDocument().addField(field);
-        VsmfieldsConfig cfg = vsmfieldsConfig(schema);
-        VsmfieldsConfig.Fieldspec fieldSpec = cfg.fieldspec().get(0);
-        assertEquals("f", fieldSpec.name());
-        assertEquals(VsmfieldsConfig.Fieldspec.Searchmethod.AUTOUTF8, fieldSpec.searchmethod());
-        assertEquals(normalize, fieldSpec.normalize());
-        assertEquals(arg1, fieldSpec.arg1());
-    }
+     private void testIndexMatching(Matching matching, VsmfieldsConfig.Fieldspec.Normalize.Enum normalize, String arg1) {
+         Schema schema = createSchema();
+         SDField field = new TemporarySDField(schema.getDocument(), "f", DataType.STRING);
+         field.parseIndexingScript("{ index }");
+         field.setMatching(matching);
+         schema.getDocument().addField(field);
+         VsmfieldsConfig cfg = vsmfieldsConfig(schema);
+         VsmfieldsConfig.Fieldspec fieldSpec = cfg.fieldspec().get(0);
+         assertEquals("f", fieldSpec.name());
+         assertEquals(VsmfieldsConfig.Fieldspec.Searchmethod.AUTOUTF8, fieldSpec.searchmethod());
+         assertEquals(normalize, fieldSpec.normalize());
+         assertEquals(arg1, fieldSpec.arg1());
+     }
 
-    @Test
-    void test_exact_string() {
-        testIndexMatching(new Matching(MatchType.TEXT),
-                          VsmfieldsConfig.Fieldspec.Normalize.LOWERCASE_AND_FOLD, "");
-        testIndexMatching(new Matching(MatchType.TEXT).setCase(Case.CASED),
-                          VsmfieldsConfig.Fieldspec.Normalize.NONE, "");
-        testIndexMatching(new Matching(MatchType.EXACT).setCase(Case.CASED),
-                          VsmfieldsConfig.Fieldspec.Normalize.LOWERCASE, "exact");
-        testIndexMatching(new Matching(MatchType.WORD),
-                          VsmfieldsConfig.Fieldspec.Normalize.LOWERCASE, "word");
-        testIndexMatching(new Matching(MatchType.WORD).setCase(Case.CASED),
-                          VsmfieldsConfig.Fieldspec.Normalize.NONE, "word");
-    }
+     @Test
+     void test_exact_string() {
+         testIndexMatching(new Matching(MatchType.TEXT),
+                 VsmfieldsConfig.Fieldspec.Normalize.LOWERCASE_AND_FOLD, "");
+         testIndexMatching(new Matching(MatchType.TEXT).setCase(Case.CASED),
+                 VsmfieldsConfig.Fieldspec.Normalize.NONE, "");
+         testIndexMatching(new Matching(MatchType.EXACT).setCase(Case.CASED),
+                 VsmfieldsConfig.Fieldspec.Normalize.LOWERCASE, "exact");
+         testIndexMatching(new Matching(MatchType.WORD),
+                 VsmfieldsConfig.Fieldspec.Normalize.LOWERCASE, "word");
+         testIndexMatching(new Matching(MatchType.WORD).setCase(Case.CASED),
+                 VsmfieldsConfig.Fieldspec.Normalize.NONE, "word");
+     }
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java
index 49ac370e763..e37999ded12 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java
@@ -50,7 +50,7 @@ public class StorageContentTest extends ContentBaseTest {
         return new VespaModelCreatorWithMockPkg(getHosts(), createStorageVespaServices(cluster1docs, cluster2docs), sds).create();
     }
 
-    public void doTestRouting(String cluster1docs, String cluster2docs, String expectedRoutes) throws Exception {
+    public void doTestRouting(String cluster1docs, String cluster2docs, String expectedRoutes) {
         VespaModel model = getStorageVespaModel(cluster1docs, cluster2docs);
 
         if (expectedRoutes == null) {
@@ -73,7 +73,6 @@ public class StorageContentTest extends ContentBaseTest {
 
         for (int i = 0; i < spec.getNumRoutes(); ++i) {
             RouteSpec r = spec.getRoute(i);
-
             routes.put(r.getName(), r);
         }