1 files changed, 11 insertions, 0 deletions
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java
index 01ffbe359f3..7ed3ab410a3 100644
--- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java
@@ -62,4 +62,15 @@ public class TokenizeTestCase {
         assertTrue(val instanceof StringFieldValue);
         assertNotNull(((StringFieldValue)val).getSpanTree(SpanTrees.LINGUISTICS));
     }
+
+    @Test
+    public void requireThatLongWordIsDropped() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("foo"));
+        new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig().setMaxTokenLength(2)).execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof StringFieldValue);
+        assertNull(((StringFieldValue)val).getSpanTree(SpanTrees.LINGUISTICS));
+    }
 }