Add a test

6 files changed, 57 insertions, 10 deletions

diff --git a/config-model/src/test/derived/tokenization/ilscripts.cfg b/config-model/src/test/derived/tokenization/ilscripts.cfg
new file mode 100644
index 00000000000..ead74110db3
--- /dev/null
+++ b/config-model/src/test/derived/tokenization/ilscripts.cfg
@@ -0,0 +1,9 @@
+maxtermoccurrences 100
+fieldmatchmaxlength 1000000
+ilscript[].doctype "tokenization"
+ilscript[].docfield[] "text"
+ilscript[].docfield[] "text_array"
+ilscript[].content[] "clear_state | guard { input text_array | for_each { lowercase } | for_each { normalize } | for_each { tokenize normalize stem:\"BEST\" } | index text_array_derived | summary text_array_derived; }"
+ilscript[].content[] "clear_state | guard { input text | normalize | tokenize normalize stem:\"BEST\" | index text_derived | summary text_derived; }"
+ilscript[].content[] "clear_state | guard { input text | tokenize normalize stem:\"BEST\" | index text | summary text; }"
+ilscript[].content[] "clear_state | guard { input text_array | for_each { tokenize normalize stem:\"BEST\" } | index text_array | summary text_array; }"
\ No newline at end of file
diff --git a/config-model/src/test/derived/tokenization/tokenization.sd b/config-model/src/test/derived/tokenization/tokenization.sd
new file mode 100644
index 00000000000..4510a574d60
--- /dev/null
+++ b/config-model/src/test/derived/tokenization/tokenization.sd
@@ -0,0 +1,23 @@
+schema tokenization {
+
+  document tokenization {
+
+    field text type string {
+      indexing: index | summary
+    }
+
+    field text_array type array<string> {
+      indexing: index | summary
+    }
+
+  }
+
+  field text_derived type string {
+    indexing: input text | normalize | index | summary
+  }
+
+  field text_array_derived type array<string> {
+    indexing: input text_array | for_each { lowercase } | for_each { normalize } | index | summary
+  }
+
+}
\ No newline at end of file
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java
new file mode 100755
index 00000000000..6fe367ef6d1
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java
@@ -0,0 +1,19 @@
+// Copyright Verizon media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.parser.ParseException;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * @author bratseh
+ */
+public class TokenizationTestCase extends AbstractExportingTestCase {
+
+    @Test
+    public void testTokenizationScripts() throws IOException, ParseException {
+        assertCorrectDeriving("tokenization");
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
index 9ae25518969..59d8a86085b 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
@@ -14,8 +14,6 @@ import com.yahoo.search.cluster.NodeManager;
 import com.yahoo.search.dispatch.TopKEstimator;
 import com.yahoo.vespa.config.search.DispatchConfig;
 
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java
index 896799d4492..9c6e698a154 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java
@@ -45,14 +45,10 @@ public final class NormalizeExpression extends Expression {
     }
 
     @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof NormalizeExpression)) {
-            return false;
-        }
-        NormalizeExpression rhs = (NormalizeExpression)obj;
-        if (linguistics != rhs.linguistics) {
-            return false;
-        }
+    public boolean equals(Object o) {
+        if (!(o instanceof NormalizeExpression)) return false;
+        NormalizeExpression other = (NormalizeExpression)o;
+        if (linguistics != other.linguistics) return false;
         return true;
     }
 
@@ -60,4 +56,5 @@ public final class NormalizeExpression extends Expression {
     public int hashCode() {
         return getClass().hashCode();
     }
+
 }
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java
index e9a0d0253e1..223c8191186 100644
--- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java
@@ -90,4 +90,5 @@ public class ExpressionTestCase {
         assertEquals(foo.hashCode(), bar.hashCode());
         assertEquals(foo, bar);
     }
+
 }