1 files changed, 17 insertions, 10 deletions
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java
index 91bd85420e0..3f2b6a5825a 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java
@@ -2,9 +2,12 @@
 package com.yahoo.vespa.indexinglanguage.expressions;
 
 import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
 import com.yahoo.document.datatypes.StringFieldValue;
 import com.yahoo.language.Language;
 import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.LinguisticsContext;
 import com.yahoo.language.process.StemMode;
 import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
 import com.yahoo.vespa.indexinglanguage.linguistics.LinguisticsAnnotator;
@@ -15,6 +18,7 @@ import com.yahoo.vespa.indexinglanguage.linguistics.LinguisticsAnnotator;
 public final class TokenizeExpression extends Expression {
 
     private final Linguistics linguistics;
+    private LinguisticsContext linguisticsContext = LinguisticsContext.empty();
     private final AnnotatorConfig config;
 
     public TokenizeExpression(Linguistics linguistics, AnnotatorConfig config) {
@@ -32,17 +36,24 @@ public final class TokenizeExpression extends Expression {
     }
 
     @Override
+    public void setStatementOutput(DocumentType documentType, Field field) {
+        linguisticsContext = new LinguisticsContext.Builder().schema(documentType.getName())
+                                                             .field( field.getName())
+                                                             .build();
+    }
+
+    @Override
     protected void doExecute(ExecutionContext context) {
         StringFieldValue input = (StringFieldValue)context.getValue();
         StringFieldValue output = input.clone();
         context.setValue(output);
 
-        AnnotatorConfig cfg = new AnnotatorConfig(config);
+        AnnotatorConfig config = new AnnotatorConfig(this.config);
         Language lang = context.resolveLanguage(linguistics);
         if (lang != null) {
-            cfg.setLanguage(lang);
+            config.setLanguage(lang);
         }
-        LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, cfg);
+        LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, linguisticsContext, config);
         annotator.annotate(output, context);
     }
 
@@ -74,13 +85,8 @@ public final class TokenizeExpression extends Expression {
 
     @Override
     public boolean equals(Object obj) {
-        if (!(obj instanceof TokenizeExpression)) {
-            return false;
-        }
-        TokenizeExpression rhs = (TokenizeExpression)obj;
-        if (!config.equals(rhs.config)) {
-            return false;
-        }
+        if ( ! (obj instanceof TokenizeExpression rhs)) return false;
+        if ( ! config.equals(rhs.config)) return false;
         return true;
     }
 
@@ -88,4 +94,5 @@ public final class TokenizeExpression extends Expression {
     public int hashCode() {
         return getClass().hashCode() + config.hashCode();
     }
+
 }