From 91ac31746b47eecf998a3d07173f0158bfeb4392 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 2 Jul 2021 12:40:30 +0200 Subject: Add a test --- .../src/test/derived/tokenization/ilscripts.cfg | 9 +++++++++ .../src/test/derived/tokenization/tokenization.sd | 23 ++++++++++++++++++++++ .../derived/TokenizationTestCase.java | 19 ++++++++++++++++++ .../dispatch/searchcluster/SearchCluster.java | 2 -- .../expressions/NormalizeExpression.java | 13 +++++------- .../parser/ExpressionTestCase.java | 1 + 6 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 config-model/src/test/derived/tokenization/ilscripts.cfg create mode 100644 config-model/src/test/derived/tokenization/tokenization.sd create mode 100755 config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java diff --git a/config-model/src/test/derived/tokenization/ilscripts.cfg b/config-model/src/test/derived/tokenization/ilscripts.cfg new file mode 100644 index 00000000000..ead74110db3 --- /dev/null +++ b/config-model/src/test/derived/tokenization/ilscripts.cfg @@ -0,0 +1,9 @@ +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "tokenization" +ilscript[].docfield[] "text" +ilscript[].docfield[] "text_array" +ilscript[].content[] "clear_state | guard { input text_array | for_each { lowercase } | for_each { normalize } | for_each { tokenize normalize stem:\"BEST\" } | index text_array_derived | summary text_array_derived; }" +ilscript[].content[] "clear_state | guard { input text | normalize | tokenize normalize stem:\"BEST\" | index text_derived | summary text_derived; }" +ilscript[].content[] "clear_state | guard { input text | tokenize normalize stem:\"BEST\" | index text | summary text; }" +ilscript[].content[] "clear_state | guard { input text_array | for_each { tokenize normalize stem:\"BEST\" } | index text_array | summary text_array; }" \ No newline at end of file diff --git a/config-model/src/test/derived/tokenization/tokenization.sd b/config-model/src/test/derived/tokenization/tokenization.sd new file mode 100644 index 00000000000..4510a574d60 --- /dev/null +++ b/config-model/src/test/derived/tokenization/tokenization.sd @@ -0,0 +1,23 @@ +schema tokenization { + + document tokenization { + + field text type string { + indexing: index | summary + } + + field text_array type array { + indexing: index | summary + } + + } + + field text_derived type string { + indexing: input text | normalize | index | summary + } + + field text_array_derived type array { + indexing: input text_array | for_each { lowercase } | for_each { normalize } | index | summary + } + +} \ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java new file mode 100755 index 00000000000..6fe367ef6d1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java @@ -0,0 +1,19 @@ +// Copyright Verizon media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author bratseh + */ +public class TokenizationTestCase extends AbstractExportingTestCase { + + @Test + public void testTokenizationScripts() throws IOException, ParseException { + assertCorrectDeriving("tokenization"); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 9ae25518969..59d8a86085b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -14,8 +14,6 @@ import com.yahoo.search.cluster.NodeManager; import com.yahoo.search.dispatch.TopKEstimator; import com.yahoo.vespa.config.search.DispatchConfig; -import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java index 896799d4492..9c6e698a154 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java @@ -45,14 +45,10 @@ public final class NormalizeExpression extends Expression { } @Override - public boolean equals(Object obj) { - if (!(obj instanceof NormalizeExpression)) { - return false; - } - NormalizeExpression rhs = (NormalizeExpression)obj; - if (linguistics != rhs.linguistics) { - return false; - } + public boolean equals(Object o) { + if (!(o instanceof NormalizeExpression)) return false; + NormalizeExpression other = (NormalizeExpression)o; + if (linguistics != other.linguistics) return false; return true; } @@ -60,4 +56,5 @@ public final class NormalizeExpression extends Expression { public int hashCode() { return getClass().hashCode(); } + } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java index e9a0d0253e1..223c8191186 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java @@ -90,4 +90,5 @@ public class ExpressionTestCase { assertEquals(foo.hashCode(), bar.hashCode()); assertEquals(foo, bar); } + } -- cgit v1.2.3