Merge pull request #19198 from vespa-engine/bratseth/il-encode

Bratseth/il encode

3 files changed, 13 insertions, 10 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
index ac92207820c..9c89517f72d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
@@ -9,6 +9,7 @@ import com.yahoo.document.MapDataType;
 import com.yahoo.document.StructDataType;
 import com.yahoo.document.TensorDataType;
 import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.Encoder;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.searchdefinition.Index;
 import com.yahoo.searchdefinition.Search;
@@ -425,12 +426,12 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
 
     /** Parse an indexing expression which will use the simple linguistics implementatino suitable for testing */
     public void parseIndexingScript(String script) {
-        parseIndexingScript(script, new SimpleLinguistics());
+        parseIndexingScript(script, new SimpleLinguistics(), Encoder.throwsOnUse);
     }
 
-    public void parseIndexingScript(String script, Linguistics linguistics) {
+    public void parseIndexingScript(String script, Linguistics linguistics, Encoder encoder) {
         try {
-            ScriptParserContext config = new ScriptParserContext(linguistics);
+            ScriptParserContext config = new ScriptParserContext(linguistics, encoder);
             config.setInputStream(new IndexingInput(script));
             setIndexingScript(ScriptExpression.newInstance(config));
         } catch (ParseException e) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
index dece0064fcc..4332d8baea8 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
@@ -2,6 +2,7 @@
 package com.yahoo.searchdefinition.fieldoperation;
 
 import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.Encoder;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.parser.ParseException;
@@ -27,14 +28,14 @@ public class IndexingOperation implements FieldOperation {
     }
 
     /** Creates an indexing operation which will use the simple linguistics implementation suitable for testing */
-    @SuppressWarnings("deprecation")
     public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine) throws ParseException {
-        return fromStream(input, multiLine, new SimpleLinguistics());
+        return fromStream(input, multiLine, new SimpleLinguistics(), Encoder.throwsOnUse);
     }
 
-    public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine, Linguistics linguistics)
+    public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine,
+                                               Linguistics linguistics, Encoder encoder)
             throws ParseException {
-        ScriptParserContext config = new ScriptParserContext(linguistics);
+        ScriptParserContext config = new ScriptParserContext(linguistics, encoder);
         config.setAnnotatorConfig(new AnnotatorConfig());
         config.setInputStream(input);
         ScriptExpression exp;
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 4d7b31e6e50..7df77588fe8 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -58,6 +58,7 @@ import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.Encoder;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.search.query.ranking.Diversity;
 import java.util.Map;
@@ -111,7 +112,7 @@ public class SDParser {
      */
     @SuppressWarnings("deprecation")
     private IndexingOperation newIndexingOperation(boolean multiline) throws ParseException {
-        return newIndexingOperation(multiline, new SimpleLinguistics());
+        return newIndexingOperation(multiline, new SimpleLinguistics(), Encoder.throwsOnUse);
     }
 
     /**
@@ -120,13 +121,13 @@ public class SDParser {
      * @param multiline Whether or not to allow multi-line expressions.
      * @param linguistics What to use for tokenizing.
      */
-    private IndexingOperation newIndexingOperation(boolean multiline, Linguistics linguistics) throws ParseException {
+    private IndexingOperation newIndexingOperation(boolean multiline, Linguistics linguistics, Encoder encoder) throws ParseException {
         SimpleCharStream input = (SimpleCharStream)token_source.input_stream;
         if (token.next != null) {
             input.backup(token.next.image.length());
         }
         try {
-            return IndexingOperation.fromStream(input, multiline, linguistics);
+            return IndexingOperation.fromStream(input, multiline, linguistics, encoder);
         } finally {
             token.next = null;
             jj_ntk = -1;